Section 5/201 menit
5. Vision Framework: Computer Vision
5. Vision Framework: Computer Vision
Vision adalah framework high-level untuk computer vision — lebih mudah dari Core ML langsung karena sudah menangani preprocessing gambar.
Built-in Vision Requests (Tanpa Model Custom)
swift
// Vision: menggunakan built-in requests Apple
import Vision
import UIKit
final class VisionAnalyzer {
// MARK: — Face Detection
func detectFaces(in image: UIImage) async throws -> [FaceInfo] {
guard let cgImage = image.cgImage else { return [] }
let request = VNDetectFaceRectanglesRequest()
let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
try handler.perform([request])
return (request.results ?? []).map { observation in
FaceInfo(
boundingBox: observation.boundingBox,
confidence: observation.confidence
)
}
}
// MARK: — Face Landmarks (mata, hidung, mulut)
func detectFaceLandmarks(in image: UIImage) async throws -> [VNFaceObservation] {
guard let cgImage = image.cgImage else { return [] }
let request = VNDetectFaceLandmarksRequest()
let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
try handler.perform([request])
return request.results ?? []
}
// MARK: — Person Segmentation (pisahkan orang dari background)
func segmentPerson(in image: UIImage) async throws -> UIImage? {
guard let cgImage = image.cgImage else { return nil }
let request = VNGeneratePersonSegmentationRequest()
request.qualityLevel = .accurate // atau .fast, .balanced
request.outputPixelFormat = kCVPixelFormatType_OneComponent8
let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
try handler.perform([request])
guard let result = request.results?.first,
let maskBuffer = result.pixelBuffer.map({ $0 }) else {
return nil
}
return UIImage(pixelBuffer: maskBuffer)
}
// MARK: — Text Recognition (OCR)
func recognizeText(in image: UIImage) async throws -> [String] {
guard let cgImage = image.cgImage else { return [] }
let request = VNRecognizeTextRequest()
request.recognitionLevel = .accurate // atau .fast
request.recognitionLanguages = ["id-ID", "en-US"]
request.usesLanguageCorrection = true
let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
try handler.perform([request])
return (request.results ?? []).compactMap { observation in
observation.topCandidates(1).first?.string
}
}
// MARK: — Barcode & QR Code
func detectBarcodes(in image: UIImage) async throws -> [BarcodeResult] {
guard let cgImage = image.cgImage else { return [] }
let request = VNDetectBarcodesRequest()
request.symbologies = [.qr, .ean13, .ean8, .code128, .dataMatrix]
let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
try handler.perform([request])
return (request.results ?? []).compactMap { observation in
guard let payload = observation.payloadStringValue else { return nil }
return BarcodeResult(symbology: observation.symbology, payload: payload)
}
}
// MARK: — Body Pose Detection
func detectBodyPose(in image: UIImage) async throws -> [VNHumanBodyPoseObservation] {
guard let cgImage = image.cgImage else { return [] }
let request = VNDetectHumanBodyPoseRequest()
let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
try handler.perform([request])
return request.results ?? []
}
// MARK: — Object Detection (perlu model custom)
func detectObjects(in image: UIImage, using model: VNCoreMLModel) async throws -> [DetectedObject] {
guard let cgImage = image.cgImage else { return [] }
return try await withCheckedThrowingContinuation { continuation in
let request = VNCoreMLRequest(model: model) { request, error in
if let error {
continuation.resume(throwing: error)
return
}
let results = (request.results as? [VNRecognizedObjectObservation] ?? [])
.filter { $0.confidence > 0.5 }
.map { obs in
DetectedObject(
label: obs.labels.first?.identifier ?? "unknown",
confidence: obs.confidence,
boundingBox: obs.boundingBox
)
}
continuation.resume(returning: results)
}
let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
do {
try handler.perform([request])
} catch {
continuation.resume(throwing: error)
}
}
}
}
struct FaceInfo {
let boundingBox: CGRect // Normalized 0-1, origin di bottom-left
let confidence: VNConfidence
}
struct BarcodeResult {
let symbology: VNBarcodeSymbology
let payload: String
}
struct DetectedObject {
let label: String
let confidence: VNConfidence
let boundingBox: CGRect
}
Real-time Vision dengan AVFoundation
swift
// Vision + AVFoundation: real-time camera analysis
import AVFoundation
import Vision
import UIKit
final class RealtimeFaceDetector: NSObject {
private let captureSession = AVCaptureSession()
private let videoOutput = AVCaptureVideoDataOutput()
private let processingQueue = DispatchQueue(label: "com.app.vision", qos: .userInteractive)
// State untuk throttling — jangan proses setiap frame
private var lastProcessedTime: CFTimeInterval = 0
private let processingInterval: CFTimeInterval = 0.1 // Proses 10fps, bukan 60fps
var onFacesDetected: (([VNFaceObservation]) -> Void)?
func startSession() throws {
captureSession.beginConfiguration()
guard let device = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .front),
let input = try? AVCaptureDeviceInput(device: device) else {
throw CameraError.deviceNotAvailable
}
captureSession.addInput(input)
captureSession.addOutput(videoOutput)
captureSession.sessionPreset = .medium // Bukan .high — hemat CPU
videoOutput.setSampleBufferDelegate(self, queue: processingQueue)
videoOutput.alwaysDiscardsLateVideoFrames = true // Drop frame jika sibuk
captureSession.commitConfiguration()
captureSession.startRunning()
}
private func processFrame(_ sampleBuffer: CMSampleBuffer) {
// Throttle: skip jika terlalu cepat
let currentTime = CACurrentMediaTime()
guard currentTime - lastProcessedTime >= processingInterval else { return }
lastProcessedTime = currentTime
guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
let request = VNDetectFaceRectanglesRequest { [weak self] request, _ in
let faces = request.results as? [VNFaceObservation] ?? []
DispatchQueue.main.async {
self?.onFacesDetected?(faces)
}
}
let handler = VNImageRequestHandler(cvPixelBuffer: pixelBuffer, orientation: .leftMirrored)
try? handler.perform([request])
}
}
extension RealtimeFaceDetector: AVCaptureVideoDataOutputSampleBufferDelegate {
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
processFrame(sampleBuffer)
}
}
enum CameraError: Error {
case deviceNotAvailable
case permissionDenied
}