Open CKey Make a CKey
Menu
Viewing is open to everyone. Downloading and starring need a GitHub sign-in. Sign in with GitHub

CameraScanner.swift

v0.1.0-build.85 Latest Commit ede38f5

Size
23 KB
Lines
428
Language
swift
SHA-256
f445edb493b8…
View on GitHub

Sign in with GitHub to download. Each download is recorded with your GitHub username.

1import AVFoundation
2import AudioToolbox
3import Foundation
4import QuartzCore
5import SwiftUI
6
7final class CameraScanner: NSObject, ObservableObject, AVCaptureVideoDataOutputSampleBufferDelegate {
8 let session = AVCaptureSession()
9 @Published private(set) var status = "Starting camera…"
10 @Published private(set) var decodedHex: String?
11 /// The payload as text, when it is valid UTF-8 with no control characters
12 /// other than tab, LF, and CR; nil for binary payloads (hex only).
13 @Published private(set) var decodedText: String?
14 /// The tappable action a text payload offers (link, email, call, map...).
15 @Published private(set) var decodedAction: PayloadAction?
16 @Published private(set) var decodedProfile = ""
17 @Published private(set) var decodeMilliseconds = 0.0
18 @Published private(set) var detectionMilliseconds = 0.0
19 @Published private(set) var candidateCount = 0
20 @Published private(set) var candidateSizes = ""
21 @Published private(set) var scanDetails = "Waiting for a camera frame."
22 @Published private(set) var isRunning = false
23
24 private let sessionQueue = DispatchQueue(label: "solutions.clearkey.ckey-reader.session")
25 private let frameQueue = DispatchQueue(label: "solutions.clearkey.ckey-reader.frames", qos: .userInitiated)
26 private let output = AVCaptureVideoDataOutput()
27 private var lastFrameTime = 0.0
28 private var lastProgressTime = 0.0
29 private var hasConfigured = false
30 private var didDecode = false
31 private var lastDecodeDiagnostic = ""
32
33 func start() {
34 switch AVCaptureDevice.authorizationStatus(for: .video) {
35 case .authorized: configureAndRun()
36 case .notDetermined:
37 AVCaptureDevice.requestAccess(for: .video) { [weak self] allowed in
38 guard let self else { return }
39 if allowed { self.configureAndRun() }
40 else { DispatchQueue.main.async { self.status = "Camera access is required to scan." } }
41 }
42 default:
43 DispatchQueue.main.async { self.status = "Enable camera access for CKey Reader in Settings." }
44 }
45 }
46
47 func stop() {
48 sessionQueue.async { [weak self] in
49 guard let self, self.session.isRunning else { return }
50 self.session.stopRunning()
51 DispatchQueue.main.async { self.isRunning = false }
52 }
53 }
54
55 func scanAgain() {
56 frameQueue.async { self.didDecode = false }
57 DispatchQueue.main.async {
58 self.decodedHex = nil
59 self.decodedText = nil
60 self.decodedAction = nil
61 self.decodedProfile = ""
62 self.status = "Looking for a CKey…"
63 }
64 }
65
66 private func configureAndRun() {
67 sessionQueue.async { [weak self] in
68 guard let self else { return }
69 if !self.hasConfigured {
70 self.session.beginConfiguration()
71 // A CKey must retain enough camera pixels per symbol unit:
72 // the core decoders need about 6 (P1), 10 (P2), and 11 (P3)
73 // px/unit. Profile 3's 116-unit field therefore needs about
74 // 1,280 px across, which a 1080-line frame cannot supply, so
75 // prefer 4K. `.high` may negotiate a smaller stream.
76 if self.session.canSetSessionPreset(.hd4K3840x2160) {
77 self.session.sessionPreset = .hd4K3840x2160
78 } else {
79 self.session.sessionPreset = self.session.canSetSessionPreset(.hd1920x1080) ? .hd1920x1080 : .high
80 }
81 // Choose the physical wide device explicitly. Selecting a
82 // virtual dual/triple device lets iOS swap focal lengths while
83 // we are estimating a projective measurement, which invalidates
84 // the scale model between adjacent frames.
85 let discovery = AVCaptureDevice.DiscoverySession(deviceTypes: [.builtInWideAngleCamera], mediaType: .video, position: .back)
86 guard let camera = discovery.devices.first,
87 let input = try? AVCaptureDeviceInput(device: camera),
88 self.session.canAddInput(input) else {
89 self.session.commitConfiguration()
90 DispatchQueue.main.async { self.status = "Could not configure the back camera." }
91 return
92 }
93 self.session.addInput(input)
94 do {
95 try camera.lockForConfiguration()
96 if camera.isFocusModeSupported(.continuousAutoFocus) {
97 camera.focusMode = .continuousAutoFocus
98 }
99 if camera.isAutoFocusRangeRestrictionSupported {
100 // The physical wide camera is already fixed at 1×, so
101 // no lens will swap. Do not force its optics to the
102 // far range: a CKey may legitimately be shown at a
103 // close, medium, or distant measured size.
104 camera.autoFocusRangeRestriction = .none
105 }
106 if camera.isExposureModeSupported(.continuousAutoExposure) {
107 camera.exposureMode = .continuousAutoExposure
108 }
109 camera.videoZoomFactor = 1.0
110 camera.isSubjectAreaChangeMonitoringEnabled = false
111 camera.unlockForConfiguration()
112 } catch { }
113 self.output.videoSettings = [
114 kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_420YpCbCr8BiPlanarFullRange
115 ]
116 self.output.alwaysDiscardsLateVideoFrames = true
117 self.output.setSampleBufferDelegate(self, queue: self.frameQueue)
118 guard self.session.canAddOutput(self.output) else {
119 self.session.commitConfiguration()
120 DispatchQueue.main.async { self.status = "Could not configure camera frames." }
121 return
122 }
123 self.session.addOutput(self.output)
124 if let connection = self.output.connection(with: .video),
125 connection.isVideoRotationAngleSupported(90) {
126 connection.videoRotationAngle = 90
127 if connection.isVideoStabilizationSupported { connection.preferredVideoStabilizationMode = .off }
128 }
129 self.session.commitConfiguration()
130 self.hasConfigured = true
131 }
132 if !self.session.isRunning { self.session.startRunning() }
133 DispatchQueue.main.async {
134 self.isRunning = true
135 self.status = "Looking for a CKey…"
136 }
137 }
138 }
139
140 func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
141 if didDecode { return }
142 let now = CACurrentMediaTime()
143 if now - lastFrameTime < 0.05 { return } // Sample up to 20 frames/s while discarding stale frames.
144 lastFrameTime = now
145 guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer),
146 CVPixelBufferGetPlaneCount(pixelBuffer) >= 1 else { return }
147
148 let started = CACurrentMediaTime()
149 CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly)
150 defer { CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly) }
151 guard let gray = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0)?.assumingMemoryBound(to: UInt8.self) else { return }
152 let width = CVPixelBufferGetWidthOfPlane(pixelBuffer, 0)
153 let height = CVPixelBufferGetHeightOfPlane(pixelBuffer, 0)
154 let stride = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0)
155
156 // A CKey's continuous outer border remains detectable after the
157 // small payload modules have blurred together at distance. Locate that
158 // high-contrast connected component first. This deliberately does not
159 // depend on a generic document-rectangle detector.
160 // Symbols may be light-on-dark or dark-on-light. Run the same
161 // geometric component pass at both polarities; the key fit later
162 // resolves the real candidate.
163 let borderQuads = borderQuads(gray, width: width, height: height, stride: stride)
164 let borderDetectionMilliseconds = (CACurrentMediaTime() - started) * 1000
165
166 // The complete locator is measured from the border component, then
167 // optimized using the printed Grid, Ring, Ray, and finder keys. No
168 // generic rectangle model or fixed distance ladder participates.
169 var liveDetails = borderQuads.isEmpty ? "Border: none" : "Border candidates: \(borderQuads.count)"
170 var lastAttemptMilliseconds = 0.0
171 for (candidateIndex, quad) in borderQuads.prefix(6).enumerated() {
172 let observed = quadPoints(quad)
173 // Compact profiles and Profile 3 have deliberately different
174 // calibration fields. Never let a strong P1/P2 score select the
175 // projective transform used to sample P3 (or vice versa).
176 let locatorQuad = refinedCKeyQuad(observed, gray: gray, width: width, height: height,
177 stride: stride, field: .locator)
178 let textQuad = refinedCKeyQuad(observed, gray: gray, width: width, height: height,
179 stride: stride, field: .text)
180 let extendedQuad = refinedCKeyQuad(observed, gray: gray, width: width, height: height,
181 stride: stride, field: .extended)
182 // Profile 0 (scalable family) first: the core aligns its frame edges,
183 // identifies the version from its Grid/Ring/Ray Keys, then decodes.
184 let scalableQuad = refinedCKeyQuad(observed, gray: gray, width: width, height: height,
185 stride: stride, field: .scalable)
186 let decodeStarted = CACurrentMediaTime()
187 if let decoded = decodeScalable(gray: gray, width: width, height: height, stride: stride, quad: scalableQuad) {
188 publishDecoded(decoded.bytes, profile: decoded.profile,
189 detectionMilliseconds: borderDetectionMilliseconds,
190 decodeMilliseconds: (CACurrentMediaTime() - decodeStarted) * 1000,
191 candidates: borderQuads.count)
192 return
193 }
194 if let decoded = decodeAnyProfile(gray: gray, width: width, height: height, stride: stride,
195 locatorQuad: locatorQuad, textQuad: textQuad, extendedQuad: extendedQuad) {
196 publishDecoded(decoded.bytes, profile: decoded.profile,
197 detectionMilliseconds: borderDetectionMilliseconds,
198 decodeMilliseconds: (CACurrentMediaTime() - decodeStarted) * 1000,
199 candidates: borderQuads.count)
200 return
201 }
202 lastAttemptMilliseconds = (CACurrentMediaTime() - decodeStarted) * 1000
203 let side = measuredSide(observed)
204 let p1Fit = calibrationScore(gray, width: width, height: height, stride: stride,
205 quad: locatorQuad, field: .locator)
206 let p2Fit = calibrationScore(gray, width: width, height: height, stride: stride,
207 quad: textQuad, field: .text)
208 let p3Fit = calibrationScore(gray, width: width, height: height, stride: stride,
209 quad: extendedQuad, field: .extended)
210 var p0Version: UInt32 = 0
211 let p0Fit = scalableQuad.count == 8 ? scalableQuad.withUnsafeBufferPointer { buffer in
212 ckey_calibration_score_scalable_gray8(gray, width, height, stride, buffer.baseAddress, &p0Version)
213 } : 0
214 // Keep the leading candidate visible in real time. It is the one
215 // the decoder is most likely to accept and makes a bad border fit
216 // obvious without waiting for a successful scan.
217 if candidateIndex == 0 {
218 liveDetails = "Border \(String(format: "%.0f", side)) px · fit P0 v\(p0Version) \(String(format: "%.1f", p0Fit)) · P1 \(String(format: "%.1f", p1Fit)) · P2 \(String(format: "%.1f", p2Fit)) · P3 \(String(format: "%.1f", p3Fit))\nAttempt \(String(format: "%.1f", lastAttemptMilliseconds)) ms · \(lastDecodeDiagnostic)"
219 }
220 }
221 DispatchQueue.main.async { self.candidateSizes = borderQuads.isEmpty ? "" : "\(borderQuads.count) measured border component(s)" }
222 publishProgress(borderQuads.isEmpty ? "Searching for CKey border and finder keys…" : "Calibrating CKey keys · \(lastDecodeDiagnostic)",
223 detectionMilliseconds: borderDetectionMilliseconds, candidates: borderQuads.count,
224 attemptMilliseconds: lastAttemptMilliseconds, details: liveDetails)
225 }
226
227 private func publishProgress(_ text: String, detectionMilliseconds: Double, candidates: Int,
228 attemptMilliseconds: Double = 0, details: String = "",
229 force: Bool = false) {
230 let now = CACurrentMediaTime()
231 guard force || now - lastProgressTime >= 0.5 else { return }
232 lastProgressTime = now
233 DispatchQueue.main.async {
234 self.status = text
235 self.detectionMilliseconds = detectionMilliseconds
236 self.decodeMilliseconds = attemptMilliseconds
237 self.candidateCount = candidates
238 if !details.isEmpty { self.scanDetails = details }
239 }
240 }
241
242 private func measuredSide(_ quad: [CGPoint]) -> Double {
243 guard quad.count == 4 else { return 0 }
244 return Double((hypot(quad[1].x - quad[0].x, quad[1].y - quad[0].y) +
245 hypot(quad[3].x - quad[0].x, quad[3].y - quad[0].y)) * 0.5)
246 }
247
248 private enum CalibrationField { case scalable, locator, text, extended }
249
250 // Fit a border to one profile's canonical field against that profile's
251 // fixed keys. Shared C++ core (ckey_refine_quad_gray8): coordinate descent
252 // over scale, rotation, and offset, then per-corner projective moves.
253 private func refinedCKeyQuad(_ observed: [CGPoint], gray: UnsafePointer<UInt8>,
254 width: Int, height: Int, stride: Int,
255 field: CalibrationField) -> [Double] {
256 guard observed.count == 4 else { return [] }
257 let border = observed.flatMap { [Double($0.x), Double($0.y)] }
258 let coreField: ckey_field
259 switch field {
260 case .scalable: coreField = CKEY_FIELD_SCALABLE
261 case .locator: coreField = CKEY_FIELD_LOCATOR
262 case .text: coreField = CKEY_FIELD_TEXT
263 case .extended: coreField = CKEY_FIELD_EXTENDED
264 }
265 var refined = [Double](repeating: 0, count: 8)
266 let score = border.withUnsafeBufferPointer { input in
267 refined.withUnsafeMutableBufferPointer { output in
268 ckey_refine_quad_gray8(gray, width, height, stride, input.baseAddress, coreField, output.baseAddress)
269 }
270 }
271 return score >= 0 ? refined : []
272 }
273
274 private func calibrationScore(_ gray: UnsafePointer<UInt8>, width: Int, height: Int,
275 stride: Int, quad: [Double], field: CalibrationField) -> Double {
276 quad.withUnsafeBufferPointer { buffer in
277 switch field {
278 case .scalable:
279 return ckey_calibration_score_scalable_gray8(gray, width, height, stride, buffer.baseAddress, nil)
280 case .locator:
281 return ckey_calibration_score_gray8(gray, width, height, stride, buffer.baseAddress)
282 case .text:
283 return ckey_calibration_score_text_gray8(gray, width, height, stride, buffer.baseAddress)
284 case .extended:
285 return ckey_calibration_score_extended_gray8(gray, width, height, stride, buffer.baseAddress)
286 }
287 }
288 }
289
290 // Profile 0 payloads reach 4,619 bytes (v31 at level M), beyond the
291 // 2,048-byte Profile 3 maximum, so this uses its own 64 KB buffer.
292 private func decodeScalable(gray: UnsafePointer<UInt8>, width: Int, height: Int, stride: Int,
293 quad: [Double]) -> (bytes: [UInt8], profile: String)? {
294 guard quad.count == 8 else { return nil }
295 var output = [UInt8](repeating: 0, count: 65535)
296 var length = 0
297 var version: UInt32 = 0
298 var layout: UInt32 = 0
299 var diagnostic = ckey_diagnostics()
300 let status = output.withUnsafeMutableBufferPointer { decoded in
301 quad.withUnsafeBufferPointer { q in
302 ckey_decode_scalable_gray8(gray, width, height, stride, q.baseAddress,
303 decoded.baseAddress, decoded.count, &length, &version, &layout, &diagnostic)
304 }
305 }
306 guard status == CKEY_OK else { return nil }
307 return (Array(output.prefix(length)), layout == 1 ? "Profile 0.1 · print · v\(version)" : "Profile 0 · v\(version)")
308 }
309
310 private func decodeAnyProfile(gray: UnsafePointer<UInt8>, width: Int, height: Int, stride: Int,
311 locatorQuad: [Double], textQuad: [Double], extendedQuad: [Double]) -> (bytes: [UInt8], profile: String)? {
312 guard locatorQuad.count == 8, textQuad.count == 8, extendedQuad.count == 8 else { return nil }
313 var locator = [UInt8](repeating: 0, count: Int(CKEY_LOCATOR_PAYLOAD_BYTES))
314 var diagnostic = ckey_diagnostics()
315 let locatorStatus = locator.withUnsafeMutableBufferPointer { output in
316 locatorQuad.withUnsafeBufferPointer { q in
317 ckey_decode_gray8(gray, width, height, stride, q.baseAddress, output.baseAddress, output.count, &diagnostic)
318 }
319 }
320 if locatorStatus == CKEY_OK { return (locator, "Profile 1 · Locator") }
321 let locatorDiagnostic = diagnosticText(diagnostic)
322
323 // Dense profiles share the actual native sampler and only succeed after
324 // their independent RS and CRC validation. They are intentionally tried
325 // after the compact profile so ordinary locator reads retain their fast path.
326 var dense = [UInt8](repeating: 0, count: Int(CKEY_EXTENDED_PAYLOAD_MAX_BYTES))
327 var length = 0
328 let textStatus = dense.withUnsafeMutableBufferPointer { output in
329 textQuad.withUnsafeBufferPointer { q in
330 ckey_decode_text_gray8(gray, width, height, stride, q.baseAddress,
331 output.baseAddress, output.count, &length, &diagnostic)
332 }
333 }
334 if textStatus == CKEY_OK { return (Array(dense.prefix(length)), "Profile 2 · Text") }
335 let textDiagnostic = diagnosticText(diagnostic)
336
337 length = 0
338 let extendedStatus = dense.withUnsafeMutableBufferPointer { output in
339 extendedQuad.withUnsafeBufferPointer { q in
340 ckey_decode_extended_gray8(gray, width, height, stride, q.baseAddress,
341 output.baseAddress, output.count, &length, &diagnostic)
342 }
343 }
344 if extendedStatus == CKEY_OK { return (Array(dense.prefix(length)), "Profile 3 · Extended") }
345 lastDecodeDiagnostic = "P1 \(locatorDiagnostic) · P2 \(textDiagnostic) · P3 \(diagnosticText(diagnostic))"
346 return nil
347 }
348
349 private func diagnosticText(_ diagnostic: ckey_diagnostics) -> String {
350 withUnsafeBytes(of: diagnostic.message) { bytes in
351 guard let address = bytes.baseAddress else { return "decoder failed" }
352 return String(cString: address.assumingMemoryBound(to: CChar.self))
353 }
354 }
355
356 static func readableText(_ payload: [UInt8]) -> String? {
357 guard !payload.isEmpty, let text = String(bytes: payload, encoding: .utf8) else { return nil }
358 let printable = text.unicodeScalars.allSatisfy { scalar in
359 // Tab, LF, and CR (vCards use CR LF line endings) are allowed.
360 scalar == "\n" || scalar == "\r" || scalar == "\t" || !CharacterSet.controlCharacters.contains(scalar)
361 }
362 return printable ? text : nil
363 }
364
365 private func publishDecoded(_ payload: [UInt8], profile: String, detectionMilliseconds: Double,
366 decodeMilliseconds: Double, candidates: Int) {
367 didDecode = true
368 let hex = payload.map { String(format: "%02x", $0) }.joined()
369 let text = CameraScanner.readableText(payload)
370 let action = text.flatMap(PayloadAction.interpret)
371 DispatchQueue.main.async {
372 self.decodedHex = hex
373 self.decodedText = text
374 self.decodedAction = action
375 self.decodedProfile = profile
376 self.detectionMilliseconds = detectionMilliseconds
377 self.candidateCount = candidates
378 self.decodeMilliseconds = decodeMilliseconds
379 self.status = "CKey decoded"
380 AudioServicesPlaySystemSound(SystemSoundID(1520))
381 }
382 }
383
384 private func quadPoints(_ quad: [Double]) -> [CGPoint] {
385 guard quad.count == 8 else { return [] }
386 return stride(from: 0, to: 8, by: 2).map {
387 CGPoint(x: CGFloat(quad[$0]), y: CGFloat(quad[$0 + 1]))
388 }
389 }
390
391 // Candidate border outlines (both polarities), most border-like first,
392 // from the shared C++ core (ckey_find_borders_gray8). Orientation comes
393 // from the border's minimum-area rectangle and each side is line-fitted,
394 // so rotated and keystoned symbols start from the right geometry.
395 private func borderQuads(_ gray: UnsafePointer<UInt8>, width: Int, height: Int,
396 stride: Int) -> [[Double]] {
397 let maxQuads = 6
398 var buffer = [Double](repeating: 0, count: maxQuads * 8)
399 let count = buffer.withUnsafeMutableBufferPointer { output in
400 ckey_find_borders_gray8(gray, width, height, stride, output.baseAddress, maxQuads)
401 }
402 return (0..<count).map { Array(buffer[($0 * 8)..<($0 * 8 + 8)]) }
403 }
404}
405
406struct CameraPreview: UIViewRepresentable {
407 let session: AVCaptureSession
408
409 func makeUIView(context: Context) -> PreviewView {
410 let view = PreviewView()
411 view.previewLayer.session = session
412 view.previewLayer.videoGravity = .resizeAspectFill
413 if let connection = view.previewLayer.connection,
414 connection.isVideoRotationAngleSupported(90) {
415 connection.videoRotationAngle = 90
416 }
417 return view
418 }
419
420 func updateUIView(_ uiView: PreviewView, context: Context) {
421 if uiView.previewLayer.session !== session { uiView.previewLayer.session = session }
422 }
423}
424
425final class PreviewView: UIView {
426 override class var layerClass: AnyClass { AVCaptureVideoPreviewLayer.self }
427 var previewLayer: AVCaptureVideoPreviewLayer { layer as! AVCaptureVideoPreviewLayer }
428}