Open CKey Make a CKey
Menu
Viewing is open to everyone. Downloading and starring need a GitHub sign-in. Sign in with GitHub

field_tests.py

v0.1.0-build.85 Latest Commit ede38f5

Size
15 KB
Lines
251
Language
python
SHA-256
0058e1f5bbd9…
View on GitHub

Sign in with GitHub to download. Each download is recorded with your GitHub username.

1#!/usr/bin/env python3
2"""Validates field-tests/suite.json and generates the files made from it.
3
4 python3 tools/field_tests.py # validate and write
5 python3 tools/field_tests.py --check # validate; fail if a generated file is stale
6 python3 tools/field_tests.py --verify-with build/ckey # encode every test code
7
8Generated: field-tests/README.md, field-tests/tests/<id>.md and
9.github/ISSUE_TEMPLATE/field-test-report.yml. openckey.org reads the same
10suite.json from the release it pins, so the site and GitHub always describe
11the same tests.
12"""
13import json
14import re
15import sys
16from pathlib import Path
17
18ROOT = Path(__file__).resolve().parent.parent
19SUITE = ROOT / "field-tests" / "suite.json"
20SITE = "https://openckey.org"
21REPO_BLOB = "https://github.com/clearkeysolutions/ckey/blob/Stable-Release/"
22FIELD_TYPES = {"text", "number", "boolean", "choice"}
23LEVELS = {"L", "M", "Q", "H"}
24LAYOUTS = {"standard", "print"}
25STYLE_KEYS = {"ink", "background", "accents", "accent_percent", "rings", "modules", "rays"}
26
27
28def fail(message):
29 sys.exit("field-tests: " + message)
30
31
32def check_field(field, where, allow_trials):
33 for key in ("id", "label", "type"):
34 if not isinstance(field.get(key), str) or not field[key]:
35 fail(f"{where}: missing {key}")
36 if not re.fullmatch(r"[a-z][a-z0-9_]{0,31}", field["id"]):
37 fail(f"{where}: id {field['id']!r} must be lower_snake_case")
38 if field["type"] not in FIELD_TYPES:
39 fail(f"{where}: type must be one of {sorted(FIELD_TYPES)}")
40 if field["type"] == "choice" and not (isinstance(field.get("options"), list) and field["options"]):
41 fail(f"{where}: a choice needs options")
42 if "trials" in field:
43 if not allow_trials:
44 fail(f"{where}: environment fields cannot have trials")
45 trials = field["trials"]
46 if not (isinstance(trials, list) and trials and len(set(trials)) == len(trials)):
47 fail(f"{where}: trials must be a non-empty list of distinct labels")
48 known = {"id", "label", "type", "required", "help", "options", "unit", "min", "max", "trials"}
49 extra = set(field) - known
50 if extra:
51 fail(f"{where}: unknown keys {sorted(extra)}")
52
53
54def validate(suite):
55 if suite.get("schema") != 1:
56 fail("schema must be 1")
57 for key in ("suite_version", "title", "summary"):
58 if not isinstance(suite.get(key), str):
59 fail(f"missing {key}")
60 ids = set()
61 for field in suite["environment"]:
62 check_field(field, "environment." + str(field.get("id")), False)
63 styles = suite.get("styles", {})
64 for name, style in styles.items():
65 extra = set(style.get("ckey_style", {})) - STYLE_KEYS
66 if extra:
67 fail(f"style {name}: unknown ckey_style keys {sorted(extra)}")
68 for test in suite["tests"]:
69 tid = test.get("id", "")
70 if not re.fullmatch(r"FT-\d{2}", tid) or tid in ids:
71 fail(f"test id {tid!r} must be FT-NN and unique")
72 ids.add(tid)
73 for key in ("title", "category", "claim", "rationale"):
74 if not isinstance(test.get(key), str) or not test[key]:
75 fail(f"{tid}: missing {key}")
76 for key in ("setup", "steps"):
77 if not (isinstance(test.get(key), list) and test[key]):
78 fail(f"{tid}: {key} must be a non-empty list")
79 if set(test.get("rubric", {})) != {"pass", "partial", "fail"}:
80 fail(f"{tid}: rubric must define pass, partial and fail")
81 labels = set()
82 for code in test["codes"]:
83 label = code.get("label")
84 if label in labels:
85 fail(f"{tid}: duplicate code label {label}")
86 labels.add(label)
87 if code.get("level") not in LEVELS or code.get("layout") not in LAYOUTS:
88 fail(f"{tid} code {label}: bad level or layout")
89 if code.get("style") not in styles:
90 fail(f"{tid} code {label}: unknown style {code.get('style')!r}")
91 size = code.get("size_in")
92 if not isinstance(size, (int, float)) or not 0.1 <= size <= 8:
93 fail(f"{tid} code {label}: size_in must be 0.1 to 8")
94 version = code.get("version", 0)
95 if not isinstance(version, int) or not 0 <= version <= 31:
96 fail(f"{tid} code {label}: version must be 0 (auto) to 31")
97 if not isinstance(code.get("payload"), str) or not code["payload"]:
98 fail(f"{tid} code {label}: missing payload")
99 mids = set()
100 for field in test["measurements"]:
101 check_field(field, f"{tid}.{field.get('id')}", True)
102 if field["id"] in mids:
103 fail(f"{tid}: duplicate measurement {field['id']}")
104 mids.add(field["id"])
105
106
107def test_md(suite, test):
108 tid = test["id"]
109 lines = [f"# {tid}: {test['title']}", "",
110 f"<!-- Generated from field-tests/suite.json by tools/field_tests.py. Edit the JSON, not this file. -->", "",
111 f"**Category:** {test['category']} · **Time:** about {test.get('time_minutes', 10)} minutes · "
112 f"**Run it and see results:** [{SITE}/tests/test?id={tid}]({SITE}/tests/test?id={tid})", "",
113 "## What it checks", "", test["claim"], "", "## Why", "", test["rationale"], "",
114 "## Test codes", "",
115 f"Print or open the sheet at [{SITE}/tests/sheet?id={tid}]({SITE}/tests/sheet?id={tid}). It is drawn by the CKey web SDK from the values below.", "",
116 "| Code | Size | Version | Level | Layout | Style | Holds |", "|---|---|---|---|---|---|---|"]
117 for code in test["codes"]:
118 version = code.get("version") or "smallest that fits"
119 lines.append(f"| {code['label']} | {code['size_in']} in | {version} | {code['level']} | {code['layout']} | {code['style']} | `{code['payload']}` |")
120 lines += ["", "## Set up", ""] + [f"{i}. {s}" for i, s in enumerate(test["setup"], 1)]
121 lines += ["", "## Steps", ""] + [f"{i}. {s}" for i, s in enumerate(test["steps"], 1)]
122 lines += ["", "## What to record", ""]
123 for field in test["measurements"]:
124 unit = f" ({field['unit']})" if field.get("unit") else ""
125 req = "required" if field.get("required") else "optional"
126 trials = f" For each of: {', '.join(field['trials'])}." if field.get("trials") else ""
127 lines.append(f"- **{field['label']}**{unit}, {field['type']}, {req}.{trials}")
128 lines += ["", "Every report also records the device, operating system, reader and its version, and the printer or screen.", "",
129 "## How to grade it", "",
130 f"- **Pass:** {test['rubric']['pass']}", f"- **Partial:** {test['rubric']['partial']}", f"- **Fail:** {test['rubric']['fail']}", "",
131 "## Report a result", "",
132 f"- On openckey.org: [{SITE}/tests/test?id={tid}]({SITE}/tests/test?id={tid}), with optional photos.",
133 f"- On GitHub: [open a field-test report]({suite['report_on_github']}&title=%5B{tid}%5D+) and choose {tid}.", "",
134 f"Both are published together at [{SITE}/tests]({SITE}/tests) and as JSON at [{suite['results_json']}]({suite['results_json']}).", ""]
135 return "\n".join(lines)
136
137
138def readme_md(suite):
139 lines = [f"# {suite['title']}", "",
140 "<!-- Generated from field-tests/suite.json by tools/field_tests.py. Edit the JSON, not this file. -->", "",
141 suite["summary"], "",
142 f"Anyone can run a test and report the result, on [{SITE}/tests]({SITE}/tests) or [here on GitHub]({suite['report_on_github']}). "
143 f"Every result is public: on the site, and as JSON at [{suite['results_json']}]({suite['results_json']}). "
144 "Results reported on GitHub are imported to the site, and each one links back to its issue.", "",
145 f"Suite version **{suite['suite_version']}**. The machine-readable definition is [`suite.json`](suite.json); "
146 "openckey.org reads it from the release it pins, so the site and this folder always match.", "",
147 "| Test | Category | What it checks |", "|---|---|---|"]
148 for test in suite["tests"]:
149 lines.append(f"| [{test['id']}: {test['title']}](tests/{test['id']}.md) | {test['category']} | {test['claim']} |")
150 lines += ["", "## Grading", "",
151 "Each test has a rubric with three outcomes. The person running the test chooses the outcome; the measurements are published beside it so anyone can check the grade.", ""]
152 for key in ("pass", "partial", "fail"):
153 lines.append(f"- **{key.capitalize()}:** {suite['outcomes'][key]}")
154 lines += ["", "## Your own tests", "",
155 f"You can also design a test on [{SITE}/tests/new]({SITE}/tests/new): its purpose, set-up, steps, what to record and its rubric. "
156 "It is listed at once, marked Community, and anyone can report results for it. "
157 "A community test that proves useful can be proposed for this suite in a pull request to `suite.json`.", "",
158 "## Photos and privacy", "",
159 "Photos sent with a result on openckey.org are checked before anyone else can see them, and a photo that shows a face waits for a person to review it. "
160 f"See [{SITE}/privacy#field-tests]({SITE}/privacy#field-tests). Photos attached to a GitHub issue are published by GitHub under its own terms.", "",
161 "## Changing the suite", "",
162 "Edit `suite.json`, run `python3 tools/field_tests.py` to regenerate this folder and the issue form, and commit both. CI runs `--check`, which fails when they differ.", ""]
163 return "\n".join(lines)
164
165
166def issue_form(suite):
167 def q(s):
168 return json.dumps(s, ensure_ascii=False)
169 options = [f"{t['id']} {t['title']}" for t in suite["tests"]] + ["Community test (give its ID below)"]
170 env = {f["id"]: f for f in suite["environment"]}
171 out = ["# Generated from field-tests/suite.json by tools/field_tests.py. Edit the JSON, not this file.",
172 "name: Field-test report",
173 "description: Report the result of a CKey field test. Results are imported to openckey.org/tests.",
174 "title: \"[FT-] \"",
175 "labels: [\"field-test\"]",
176 "body:",
177 " - type: markdown",
178 " attributes:",
179 " value: " + q(f"Run a test from {SITE}/tests (each test page has its set-up, steps, sheet and rubric), then report it here. This report is public and is copied to {SITE}/tests with a link back to this issue."),
180 " - type: dropdown", " id: test", " attributes:", " label: " + q('Test'), " options:"]
181 out += [f" - {q(o)}" for o in options]
182 out += [" validations:", " required: true",
183 " - type: input", " id: community_test", " attributes:", " label: " + q('Community test ID'),
184 " description: " + q('Only for a community test, for example C-7Q2K9P.'), " validations:", " required: false"]
185 for fid in ("device", "os", "reader_version", "medium"):
186 f = env[fid]
187 out += [" - type: input", f" id: {fid}", " attributes:", f" label: {q(f['label'])}"]
188 if f.get("help"):
189 out.append(f" description: {q(f['help'])}")
190 out += [" validations:", f" required: {'true' if f.get('required') else 'false'}"]
191 out += [" - type: dropdown", " id: reader", " attributes:", f" label: {q(env['reader']['label'])}", " options:"]
192 out += [f" - {q(o)}" for o in env["reader"]["options"]]
193 out += [" validations:", " required: true",
194 " - type: textarea", " id: results", " attributes:", " label: " + q('Measurements'),
195 " description: " + q('One line per trial, as the test page lists them, for example `A: read, 1.2 s` or `60 cm: no`.'),
196 " validations:", " required: true",
197 " - type: dropdown", " id: outcome", " attributes:", " label: " + q('Outcome'),
198 " description: " + q("Grade it with the test's rubric."), " options:",
199 " - \"pass\"", " - \"partial\"", " - \"fail\"", " validations:", " required: true",
200 " - type: textarea", " id: notes", " attributes:", " label: " + q('Notes and photos'),
201 " description: " + q('Anything else worth knowing. You can attach photos; do not include faces or personal information.'),
202 " validations:", " required: false", ""]
203 return "\n".join(out)
204
205
206def verify_codes(suite, cli):
207 """Encodes every test code with the ckey command-line tool, so a payload that no longer fits its version fails CI."""
208 import subprocess
209 import tempfile
210 with tempfile.TemporaryDirectory() as tmp:
211 for test in suite["tests"]:
212 for code in test["codes"]:
213 args = [cli, "encode-scalable", "--utf8", code["payload"], "--level", code["level"],
214 "--layout", code["layout"], "--out", str(Path(tmp) / "code.svg")]
215 if code.get("version"):
216 args += ["--version", str(code["version"])]
217 result = subprocess.run(args, capture_output=True, text=True)
218 if result.returncode != 0:
219 fail(f"{test['id']} code {code['label']} does not encode: {(result.stderr or result.stdout).strip()}")
220 print("field-tests: every test code encodes")
221
222
223def main():
224 suite = json.loads(SUITE.read_text())
225 validate(suite)
226 if "--verify-with" in sys.argv:
227 verify_codes(suite, sys.argv[sys.argv.index("--verify-with") + 1])
228 return
229 outputs = {ROOT / "field-tests" / "README.md": readme_md(suite),
230 ROOT / ".github" / "ISSUE_TEMPLATE" / "field-test-report.yml": issue_form(suite)}
231 for test in suite["tests"]:
232 outputs[ROOT / "field-tests" / "tests" / f"{test['id']}.md"] = test_md(suite, test)
233 expected = {p.name for p in outputs if p.parent.name == "tests"}
234 stale_extra = [p for p in (ROOT / "field-tests" / "tests").glob("*.md") if p.name not in expected]
235 if "--check" in sys.argv:
236 stale = [str(p.relative_to(ROOT)) for p, text in outputs.items() if not p.exists() or p.read_text() != text]
237 stale += [str(p.relative_to(ROOT)) for p in stale_extra]
238 if stale:
239 fail("generated files are stale; run python3 tools/field_tests.py: " + ", ".join(stale))
240 print(f"field-tests: {len(suite['tests'])} tests valid; generated files current")
241 return
242 for path, text in outputs.items():
243 path.parent.mkdir(parents=True, exist_ok=True)
244 path.write_text(text)
245 for path in stale_extra:
246 path.unlink()
247 print(f"field-tests: wrote {len(outputs)} files for {len(suite['tests'])} tests")
248
249
250if __name__ == "__main__":
251 main()