"""Privacy check (CM-25, 2026-09-26): read-only. Fails if any setting that https://comp-maps.ca/privacy.html depends on has drifted, so no deploy goes out over a page that has quietly become false. Run by the deploy scripts before they change anything; also runnable alone: python privacy_check.py Changing one of these settings on purpose means changing the privacy page first, then this list (Hosting/DEPLOY.md, step 11).""" # ruff: noqa: PLR2004 - the expected values are the checklist itself import ast import json import pathlib import re import sys import boto3 R = "ca-central-1" ACCT = "" HERE = pathlib.Path(__file__).resolve().parent API_SRC = HERE / "api_handler.py" # the job API, kept alongside on the operator's computer logs = boto3.client("logs", region_name=R) ddb = boto3.client("dynamodb", region_name=R) s3 = boto3.client("s3", region_name=R) sqs = boto3.client("sqs", region_name=R) apigw = boto3.client("apigatewayv2", region_name=R) trail = boto3.client("cloudtrail", region_name=R) cf = boto3.client("cloudfront") failures = [] def check(what, ok): print(("ok " if ok else "DRIFT ") + what) if not ok: failures.append(what) def lifecycle(bucket): try: rules = s3.get_bucket_lifecycle_configuration(Bucket=bucket)["Rules"] except Exception: # noqa: BLE001 return {} return {r["ID"]: r.get("Expiration", {}).get("Days") for r in rules if r["Status"] == "Enabled"} g = logs.describe_log_groups(logGroupNamePrefix="/aws/lambda/comp-map-worker")["logGroups"] check("worker logs kept 7 days", bool(g) and g[0].get("retentionInDays") == 7) for table in ("site-maps-jobs", "site-maps-terms-acceptances"): ttl = ddb.describe_time_to_live(TableName=table)["TimeToLiveDescription"] check( f"{table}: expiry on (expires_at)", ttl.get("TimeToLiveStatus") == "ENABLED" and ttl.get("AttributeName") == "expires_at", ) pitr = ddb.describe_continuous_backups(TableName="site-maps-jobs")["ContinuousBackupsDescription"] check( "site-maps-jobs: point-in-time recovery off", pitr["PointInTimeRecoveryDescription"]["PointInTimeRecoveryStatus"] == "DISABLED", ) check( "site-maps-packages: versioning never on", "Status" not in s3.get_bucket_versioning(Bucket="site-maps-packages"), ) lc = lifecycle("site-maps-packages") check("maps expire at 30 days", lc.get("compmaps-30d") == 30) check("geocode cache expires at 7 days", lc.get("geocache-7d") == 7) check( "tile cache rules (street map 7 days, aerial 30)", lc.get("tilecache-osm-7d") == 7 and lc.get("tilecache-imagery-30d") == 30, ) deep = [ o["Key"] for pg in s3.get_paginator("list_objects_v2").paginate( Bucket="site-maps-packages", Prefix="tilecache/" ) for o in pg.get("Contents", []) if int(o["Key"].split("/")[2]) > 14 ] # noqa: PLR2004 check("no close-zoom tile stored (zoom 15 or closer)", not deep) # CM-41 (2026-10-04, IP Issues' proposal): listing is how a stored tile set would be # read back as a record of where maps were drawn. Only the operator (who runs this # check) and SiteMaps' API may list the bucket; the API's own IAM policy limits it to # packages/* (it builds the package page's preview list), so it can never list the # tile cache. The second exemption was added live by Main Dev on 2026-10-04 at 17:36, # after the operator-only version blanked SiteMaps' previews. The audit trail must not # log tile reads or writes either, or it would keep their key names for 7 years. EXEMPT = {f"arn:aws:iam::{ACCT}:user/admin-vince", f"arn:aws:iam::{ACCT}:role/site-maps-api-role"} try: pol = json.loads(s3.get_bucket_policy(Bucket="site-maps-packages")["Policy"]) except Exception: # noqa: BLE001 - no policy at all pol = {"Statement": []} def _exempt(st): a = st.get("Action") if st.get("Effect") != "Deny" or "s3:ListBucket" not in (a if isinstance(a, list) else [a]): return None v = st.get("Condition", {}).get("ArnNotEquals", {}).get("aws:PrincipalArn") return set(v if isinstance(v, list) else [v]) if v else None found = [e for e in (_exempt(st) for st in pol.get("Statement", [])) if e is not None] check( "site-maps-packages: listing denied to all but the operator and SiteMaps' API", any(e == EXEMPT for e in found), ) iam = boto3.client("iam") leaks = [] for arn in sorted(a for a in EXEMPT if ":role/" in a): for pre in ("", "tilecache/", "tilecache/osm/14/"): kw = {} if pre: # with the prefix filled in, or a prefix-conditioned grant hides (CM-41) kw["ContextEntries"] = [ { "ContextKeyName": "s3:prefix", "ContextKeyValues": [pre], "ContextKeyType": "string", } ] d = iam.simulate_principal_policy( PolicySourceArn=arn, ActionNames=["s3:ListBucket"], ResourceArns=["arn:aws:s3:::site-maps-packages"], **kw, )["EvaluationResults"][0]["EvalDecision"] if d == "allowed": leaks.append(f"{arn.rsplit('/', 1)[1]} [{pre or 'no prefix'}]") detail = f" ({', '.join(leaks)})" if leaks else "" check(f"no exempt role can list the tile cache{detail}", not leaks) check("audit log kept 7 years", lifecycle(f"comp-maps-audit-{ACCT}").get("audit-7y") == 2557) # Git Manager, 5 Oct 2026 (Vince approved): a deleted or replaced audit file is kept # 35 days, then removed; the shared terms table cannot be deleted and recovers 35 days. AUDIT = f"comp-maps-audit-{ACCT}" check( "audit bucket: versioning on", s3.get_bucket_versioning(Bucket=AUDIT).get("Status") == "Enabled", ) try: _rules = s3.get_bucket_lifecycle_configuration(Bucket=AUDIT)["Rules"] except Exception: # noqa: BLE001 _rules = [] check( "audit bucket: replaced or deleted files kept 35 days, then removed", any( r["ID"] == "removed-after-35d" and r["Status"] == "Enabled" and r.get("NoncurrentVersionExpiration", {}).get("NoncurrentDays") == 35 for r in _rules ), ) _terms = ddb.describe_table(TableName="site-maps-terms-acceptances")["Table"] check( "site-maps-terms-acceptances: deletion protection on", _terms.get("DeletionProtectionEnabled") is True, ) _tpitr = ddb.describe_continuous_backups(TableName="site-maps-terms-acceptances")[ "ContinuousBackupsDescription" ]["PointInTimeRecoveryDescription"] check( "site-maps-terms-acceptances: point-in-time recovery on, 35 days", _tpitr["PointInTimeRecoveryStatus"] == "ENABLED" and _tpitr.get("RecoveryPeriodInDays", 35) == 35, ) for q in ("comp-map-jobs", "comp-map-jobs-dlq"): url = f"https://sqs.{R}.amazonaws.com/{ACCT}/{q}" days = int( sqs.get_queue_attributes(QueueUrl=url, AttributeNames=["MessageRetentionPeriod"])[ "Attributes" ]["MessageRetentionPeriod"] ) check(f"{q}: messages kept 1 day at most", days <= 86400) stages = apigw.get_stages(ApiId="miro8ndp7h")["Items"] check("API access logs off", all(not s.get("AccessLogSettings") for s in stages)) status = trail.get_trail_status(Name="comp-maps-audit") check("audit trail logging", status.get("IsLogging") is True) sel = trail.get_event_selectors(TrailName="comp-maps-audit").get("AdvancedEventSelectors", []) s3sel = [ f for s in sel for f in s["FieldSelectors"] if any("compmaps/" in v for v in f.get("StartsWith", [])) ] excl = [f for s in sel for f in s["FieldSelectors"] if f.get("Field") == "userIdentity.arn"] check("audit trail covers the maps", bool(s3sel)) tile_arn = "arn:aws:s3:::site-maps-packages/tilecache/osm/14/0/0" tile_logged = [ s.get("Name", "") for s in sel if any( f.get("Field") == "eventCategory" and "Data" in f.get("Equals", []) for f in s["FieldSelectors"] ) and all( not f.get("StartsWith") or any(tile_arn.startswith(v) for v in f["StartsWith"]) for f in s["FieldSelectors"] if f.get("Field") == "resources.ARN" ) and any( f.get("Field") == "resources.type" and "AWS::S3::Object" in f.get("Equals", []) for f in s["FieldSelectors"] ) ] check("audit trail does not log the tile cache", not tile_logged) check( "audit trail skips users' reads (no user IP addresses)", any( v.startswith(f"arn:aws:sts::{ACCT}:assumed-role/site-maps-api-role/") for f in excl for v in f.get("NotStartsWith", []) ), ) src = (HERE / "handler.py").read_text(encoding="utf-8") check("email subject names no address", '"Subject": {"Data": "Your CompMaps are ready"}' in src) check( "worker does not echo the console to its log", "if LOCAL:\n self.real.write(s)" in src, ) tiles_src = (HERE.parent / "compmap" / "tiles.py").read_text(encoding="utf-8") check( "tile cache stops at zoom 14", "MAX_CACHED_ZOOM = 14" in tiles_src and "zoom <= MAX_CACHED_ZOOM" in tiles_src, ) # One store path (SiteMaps lesson 4): S3 writes only in the routines that lock or # that hold nothing about a property. ALLOWED_WRITERS = {"_cache_put", "put", "_store", "_store_diagnostics"} writers = set() for node in ast.walk(ast.parse(src)): if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): for sub in ast.walk(node): if isinstance(sub, ast.Attribute) and sub.attr in ( "put_object", "upload_file", "upload_fileobj", ): writers.add(node.name) check( "S3 writes only in the known routines", writers <= ALLOWED_WRITERS, ) if not writers <= ALLOWED_WRITERS: print(" unexpected writers:", sorted(writers - ALLOWED_WRITERS)) # Every attribute a CompMaps job row keeps (SiteMaps lesson 1): nothing new may appear. ROW_FIELDS = { "job_id", "tool", "status", "address", "email", "created_at", "expires_at", "access_hash", "enc_summary", "enc_log", "enc_error_public", "maps", "diag_key", "error", } jobs = boto3.resource("dynamodb", region_name=R).Table("site-maps-jobs") rows, kw = [], {"FilterExpression": "tool = :c", "ExpressionAttributeValues": {":c": "compmap"}} while True: page = jobs.scan(**kw) rows += page["Items"] if "LastEvaluatedKey" not in page: break kw["ExclusiveStartKey"] = page["LastEvaluatedKey"] extra = sorted({k for r in rows for k in r} - ROW_FIELDS) check(f"CompMaps job rows hold only known fields ({len(rows)} rows)", not extra) if extra: print(" unexpected fields:", extra) check( "row labels name no address", all( re.fullmatch(r"Comp Map \(\d+ rows\)", str(r.get("address", "Comp Map (0 rows)"))) for r in rows ), ) check( "row errors are a type name only", all(re.fullmatch(r"\w*", str(r.get("error", ""))) for r in rows), ) api = API_SRC.read_text(encoding="utf-8") if API_SRC.exists() else "" check( "API sets expiry on job rows and terms acceptances", '"expires_at": now + COMP_MAP_KEEP_DAYS * 86400' in api and '"expires_at": int(time.time()) + TERMS_KEEP_DAYS * 86400' in api, ) # CM-44 (2026-11-01): firm accounts. The styles bucket is private; the firm table holds # only a code's hash, a firm's short name, a status and a paid-to date; the API logs # no CompMaps code (Main Dev's condition, 31 Oct 2026). pab = s3.get_public_access_block(Bucket="comp-maps-styles")["PublicAccessBlockConfiguration"] check("comp-maps-styles: all public access blocked", len(pab) == 4 and all(pab.values())) check( "comp-maps-styles: versioning on", s3.get_bucket_versioning(Bucket="comp-maps-styles").get("Status") == "Enabled", ) fpitr = ddb.describe_continuous_backups(TableName="comp-maps-firms")["ContinuousBackupsDescription"] check( "comp-maps-firms: point-in-time recovery off (deleted accounts leave no backup)", fpitr["PointInTimeRecoveryDescription"]["PointInTimeRecoveryStatus"] == "DISABLED", ) FIRM_FIELDS = {"code_hash", "firm_id", "status", "paid_to"} firms_t = boto3.resource("dynamodb", region_name=R).Table("comp-maps-firms") firm_rows, kw = [], {} while True: page = firms_t.scan(**kw) firm_rows += page["Items"] if "LastEvaluatedKey" not in page: break kw["ExclusiveStartKey"] = page["LastEvaluatedKey"] extra = sorted({k for r in firm_rows for k in r} - FIRM_FIELDS) check(f"comp-maps-firms holds only known fields ({len(firm_rows)} firms)", not extra) check( "firm codes stored only as hashes", all(re.fullmatch(r"[0-9a-f]{64}", str(r.get("code_hash", ""))) for r in firm_rows), ) cm_code = "" if "def _compmap_code" in api: cm_code = api.split("def _compmap_code")[1].split("\ndef ")[0] check( "API logs no CompMaps code", not cm_code or ("[compmap] rejected code" in cm_code and "%r" not in cm_code), ) # CM-27: the security headers stay on the site, and the live policy matches the file in the repo. want = __import__("json").loads( (HERE / "aws" / "cf-response-headers-policy.json").read_text(encoding="utf-8") ) dist = cf.get_distribution_config(Id="EG23C0YDL57E8")["DistributionConfig"] pid = dist["DefaultCacheBehavior"].get("ResponseHeadersPolicyId", "") live = ( cf.get_response_headers_policy(Id=pid)["ResponseHeadersPolicy"]["ResponseHeadersPolicyConfig"] if pid else {} ) sh, wsh = live.get("SecurityHeadersConfig", {}), want["SecurityHeadersConfig"] def _hdrs(cfg, key): return sorted(json.dumps(i, sort_keys=True) for i in cfg.get(key, {}).get("Items", [])) check( "site carries the security headers (CSP, HSTS, no-referrer, permissions, COOP, CORP, no Server) as in aws/cf-response-headers-policy.json", bool(pid) and sh.get("ContentSecurityPolicy", {}).get("ContentSecurityPolicy") == wsh["ContentSecurityPolicy"]["ContentSecurityPolicy"] and sh.get("ReferrerPolicy", {}).get("ReferrerPolicy") == "no-referrer" and sh.get("StrictTransportSecurity", {}).get("AccessControlMaxAgeSec", 0) >= 31536000 and _hdrs(live, "CustomHeadersConfig") == _hdrs(want, "CustomHeadersConfig") and _hdrs(live, "RemoveHeadersConfig") == _hdrs(want, "RemoveHeadersConfig"), ) # CM-30: the security@ inbox keeps incoming mail 7 days, and its log holds no addresses. check( "security inbox mail expires at 7 days", lifecycle(f"comp-maps-mail-{ACCT}").get("inbound-7d") == 7, ) g = logs.describe_log_groups(logGroupNamePrefix="/aws/lambda/mail-forwarder")["logGroups"] check("security inbox log kept 7 days", bool(g) and g[0].get("retentionInDays") == 7) check( "security inbox bucket unversioned", s3.get_bucket_versioning(Bucket=f"comp-maps-mail-{ACCT}").get("Status") != "Enabled", ) # CM-30: the security contact file must not lapse (its Expires date is required, a year at most). _sec = HERE / "site" / ".well-known" / "security.txt" _exp = ( re.search(r"^Expires: (\S+)", _sec.read_text(encoding="utf-8"), re.MULTILINE) if _sec.exists() else None ) check( "security.txt present and more than 30 days from expiry", bool(_exp) and ( __import__("datetime").datetime.fromisoformat(_exp.group(1).replace("Z", "+00:00")) - __import__("datetime").datetime.now(__import__("datetime").UTC) ).days > 30, ) # CM-26 step 1: the code the privacy page publishes is the code that runs. sys.path.insert(0, str(HERE)) from make_public_source import build # noqa: E402 pub = HERE / "site" / "source" check( "published code (site/source) matches the running code", all( (pub / n).exists() and (pub / n).read_text(encoding="utf-8") == t for n, t in build().items() ), ) print(f"\n{'PRIVACY CHECK FAILED: ' + '; '.join(failures) if failures else 'privacy check passed'}") sys.exit(1 if failures else 0)