Section 5/201 menit

5. Vision Framework: Computer Vision

5. Vision Framework: Computer Vision

Vision adalah framework high-level untuk computer vision — lebih mudah dari Core ML langsung karena sudah menangani preprocessing gambar.

Built-in Vision Requests (Tanpa Model Custom)

swift
// Vision: menggunakan built-in requests Apple

import Vision
import UIKit

final class VisionAnalyzer {

    // MARK: — Face Detection

    func detectFaces(in image: UIImage) async throws -> [FaceInfo] {
        guard let cgImage = image.cgImage else { return [] }

        let request = VNDetectFaceRectanglesRequest()
        let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
        try handler.perform([request])

        return (request.results ?? []).map { observation in
            FaceInfo(
                boundingBox: observation.boundingBox,
                confidence: observation.confidence
            )
        }
    }

    // MARK: — Face Landmarks (mata, hidung, mulut)

    func detectFaceLandmarks(in image: UIImage) async throws -> [VNFaceObservation] {
        guard let cgImage = image.cgImage else { return [] }

        let request = VNDetectFaceLandmarksRequest()
        let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
        try handler.perform([request])
        return request.results ?? []
    }

    // MARK: — Person Segmentation (pisahkan orang dari background)

    func segmentPerson(in image: UIImage) async throws -> UIImage? {
        guard let cgImage = image.cgImage else { return nil }

        let request = VNGeneratePersonSegmentationRequest()
        request.qualityLevel = .accurate  // atau .fast, .balanced
        request.outputPixelFormat = kCVPixelFormatType_OneComponent8

        let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
        try handler.perform([request])

        guard let result = request.results?.first,
              let maskBuffer = result.pixelBuffer.map({ $0 }) else {
            return nil
        }

        return UIImage(pixelBuffer: maskBuffer)
    }

    // MARK: — Text Recognition (OCR)

    func recognizeText(in image: UIImage) async throws -> [String] {
        guard let cgImage = image.cgImage else { return [] }

        let request = VNRecognizeTextRequest()
        request.recognitionLevel = .accurate  // atau .fast
        request.recognitionLanguages = ["id-ID", "en-US"]
        request.usesLanguageCorrection = true

        let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
        try handler.perform([request])

        return (request.results ?? []).compactMap { observation in
            observation.topCandidates(1).first?.string
        }
    }

    // MARK: — Barcode & QR Code

    func detectBarcodes(in image: UIImage) async throws -> [BarcodeResult] {
        guard let cgImage = image.cgImage else { return [] }

        let request = VNDetectBarcodesRequest()
        request.symbologies = [.qr, .ean13, .ean8, .code128, .dataMatrix]

        let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
        try handler.perform([request])

        return (request.results ?? []).compactMap { observation in
            guard let payload = observation.payloadStringValue else { return nil }
            return BarcodeResult(symbology: observation.symbology, payload: payload)
        }
    }

    // MARK: — Body Pose Detection

    func detectBodyPose(in image: UIImage) async throws -> [VNHumanBodyPoseObservation] {
        guard let cgImage = image.cgImage else { return [] }

        let request = VNDetectHumanBodyPoseRequest()
        let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
        try handler.perform([request])
        return request.results ?? []
    }

    // MARK: — Object Detection (perlu model custom)

    func detectObjects(in image: UIImage, using model: VNCoreMLModel) async throws -> [DetectedObject] {
        guard let cgImage = image.cgImage else { return [] }

        return try await withCheckedThrowingContinuation { continuation in
            let request = VNCoreMLRequest(model: model) { request, error in
                if let error {
                    continuation.resume(throwing: error)
                    return
                }

                let results = (request.results as? [VNRecognizedObjectObservation] ?? [])
                    .filter { $0.confidence > 0.5 }
                    .map { obs in
                        DetectedObject(
                            label: obs.labels.first?.identifier ?? "unknown",
                            confidence: obs.confidence,
                            boundingBox: obs.boundingBox
                        )
                    }

                continuation.resume(returning: results)
            }

            let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
            do {
                try handler.perform([request])
            } catch {
                continuation.resume(throwing: error)
            }
        }
    }
}

struct FaceInfo {
    let boundingBox: CGRect  // Normalized 0-1, origin di bottom-left
    let confidence: VNConfidence
}

struct BarcodeResult {
    let symbology: VNBarcodeSymbology
    let payload: String
}

struct DetectedObject {
    let label: String
    let confidence: VNConfidence
    let boundingBox: CGRect
}

Real-time Vision dengan AVFoundation

swift
// Vision + AVFoundation: real-time camera analysis

import AVFoundation
import Vision
import UIKit

final class RealtimeFaceDetector: NSObject {
    private let captureSession = AVCaptureSession()
    private let videoOutput = AVCaptureVideoDataOutput()
    private let processingQueue = DispatchQueue(label: "com.app.vision", qos: .userInteractive)

    // State untuk throttling — jangan proses setiap frame
    private var lastProcessedTime: CFTimeInterval = 0
    private let processingInterval: CFTimeInterval = 0.1  // Proses 10fps, bukan 60fps

    var onFacesDetected: (([VNFaceObservation]) -> Void)?

    func startSession() throws {
        captureSession.beginConfiguration()

        guard let device = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .front),
              let input = try? AVCaptureDeviceInput(device: device) else {
            throw CameraError.deviceNotAvailable
        }

        captureSession.addInput(input)
        captureSession.addOutput(videoOutput)
        captureSession.sessionPreset = .medium  // Bukan .high — hemat CPU

        videoOutput.setSampleBufferDelegate(self, queue: processingQueue)
        videoOutput.alwaysDiscardsLateVideoFrames = true  // Drop frame jika sibuk

        captureSession.commitConfiguration()
        captureSession.startRunning()
    }

    private func processFrame(_ sampleBuffer: CMSampleBuffer) {
        // Throttle: skip jika terlalu cepat
        let currentTime = CACurrentMediaTime()
        guard currentTime - lastProcessedTime >= processingInterval else { return }
        lastProcessedTime = currentTime

        guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }

        let request = VNDetectFaceRectanglesRequest { [weak self] request, _ in
            let faces = request.results as? [VNFaceObservation] ?? []
            DispatchQueue.main.async {
                self?.onFacesDetected?(faces)
            }
        }

        let handler = VNImageRequestHandler(cvPixelBuffer: pixelBuffer, orientation: .leftMirrored)
        try? handler.perform([request])
    }
}

extension RealtimeFaceDetector: AVCaptureVideoDataOutputSampleBufferDelegate {
    func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
        processFrame(sampleBuffer)
    }
}

enum CameraError: Error {
    case deviceNotAvailable
    case permissionDenied
}