Source enlisted AFSC data from the official DAFECD (versioned, as-of lookup)

.github/workflows/main.yml

@@ -16,6 +16,7 @@ jobs:
         ruby:
           - "3.3.8"
           - "3.4.3"
+          - "4.0.5"
 
     steps:
       - uses: actions/checkout@v6

.gitignore

@@ -6,3 +6,7 @@
 /pkg/
 /spec/reports/
 /tmp/
+
+# Source classification directories (DAFECD/DAFOCD) are extraction inputs,
+# not shipped data. Keep them out of the repo.
+*.pdf

.tool-versions

@@ -1 +1 @@
-ruby 3.4.3
+ruby 4.0.5

Gemfile

@@ -18,3 +18,5 @@ gem "simplecov"
 gem "reissue"
 
 gem "nokogiri"
+
+gem "pdf-reader"

Gemfile.lock

@@ -6,13 +6,17 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
+    Ascii85 (2.0.1)
+    afm (1.0.0)
     ast (2.4.3)
+    bigdecimal (3.3.1)
     date (3.5.1)
     debug (1.11.1)
       irb (~> 1.10)
       reline (>= 0.3.8)
     docile (1.4.1)
     erb (6.0.1)
+    hashery (2.1.2)
     io-console (0.8.2)
     irb (1.16.0)
       pp (>= 0.6.0)
@@ -31,6 +35,12 @@ GEM
     parser (3.3.10.1)
       ast (~> 2.4.1)
       racc
+    pdf-reader (2.15.1)
+      Ascii85 (>= 1.0, < 3.0, != 2.0.0)
+      afm (>= 0.2.1, < 2)
+      hashery (~> 2.0)
+      ruby-rc4
+      ttfunk
     pp (0.6.3)
       prettyprint
     prettyprint (0.2.0)
@@ -69,6 +79,7 @@ GEM
       rubocop (>= 1.75.0, < 2.0)
       rubocop-ast (>= 1.47.1, < 2.0)
     ruby-progressbar (1.13.0)
+    ruby-rc4 (0.1.5)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -89,6 +100,8 @@ GEM
       rubocop-performance (~> 1.26.0)
     stringio (3.2.0)
     tsort (0.2.0)
+    ttfunk (1.8.0)
+      bigdecimal (~> 3.1)
     unicode-display_width (3.2.0)
       unicode-emoji (~> 4.1)
     unicode-emoji (4.2.0)
@@ -102,6 +115,7 @@ DEPENDENCIES
   irb
   minitest
   nokogiri
+  pdf-reader
   rake
   reissue
   simplecov

README.md

@@ -31,12 +31,34 @@ require 'gov_codes/afsc'
 
 # Find an enlisted AFSC code
 code = GovCodes::AFSC.find("1A1X2")
-puts code.name # => "Mobility force aviator"
+puts code.name # => "Mobility Force Aviator"
 puts code.career_field # => "1A"
 puts code.career_field_subdivision # => "1A1"
 puts code.skill_level # => "X"
 puts code.specific_afsc # => "1A1X2"
 puts code.shredout # => nil
+puts code.effective_date # => #<Date: 2025-10-31>
+
+# Look up the concrete code an HR system actually stores
+# (the "7" is the airman's skill level: Craftsman)
+code = GovCodes::AFSC.find("1A172Y")
+code.specialty_name     # => "Mobility Force Aviator"
+code.skill_level_number # => 7
+code.skill_level_name   # => "Craftsman" (title comes from the directory)
+code.specialty          # => :"1A1X2"
+
+# shredout_name resolves the shredout's meaning when that shredout
+# exists in the data (nil otherwise)
+code = GovCodes::AFSC.find("1A172A")
+code.shredout_name      # => "C-5 Flight Engineer"
+
+# Enlisted lookups are versioned by DAFECD release (published each 30 Apr and
+# 31 Oct). `find`/`search` default to the latest shipped release; pass `as_of:`
+# (a Date or "YYYY-MM-DD" string) to resolve the release in effect on that date.
+code = GovCodes::AFSC.find("1A172Y", as_of: "2025-11-01")
+code.effective_date     # => #<Date: 2025-10-31>
+# A date before the earliest shipped release has no data and returns nil.
+GovCodes::AFSC.find("1A172Y", as_of: "2000-01-01") # => nil
 
 # Find an officer AFSC code
 code = GovCodes::AFSC.find("11MX")
@@ -72,9 +94,9 @@ results.each do |code|
 end
 # Output:
 # 1Z1X1: Pararescue
-# 1Z2X1: Combat control
-# 1Z3X1: Tactical air control party (TACP)
-# 1Z4X1: Special reconnaissance
+# 1Z2X1: Combat Control
+# 1Z3X1: Tactical Air Control Party (TACP)
+# 1Z4X1: Special Reconnaissance
 
 # Search for Bomber Pilot shredouts
 results = GovCodes::AFSC.search("11BX")
@@ -95,21 +117,56 @@ GovCodes::AFSC.search("1z1") # Same as search("1Z1")
 
 ### Extending with Custom AFSC Codes
 
-You can extend the default AFSC codes with your own custom codes by placing a YAML file in your application's load path:
+Enlisted codes are stored as a specialty-keyed index per DAFECD release. You can
+extend or override a release by dropping an index file for that release's
+effective date onto your application's load path:
+
+```yaml
+# In your application's lib/gov_codes/afsc/releases/dafecd/2025-10-31/enlisted.yml
+:"9Z9X9":
+  :name: Custom Specialty
+  :career_field: :"9Z"
+  :skill_levels:
+    7:
+      :code: 9Z979
+      :title: Craftsman
+  :shredouts:
+    :A: Custom Shredout
+```
+
+The gem merges your index over the shipped index for the matching release,
+adding new specialties and overriding existing ones.
+
+You can also add a whole new release (e.g. a newer directory the gem has not
+shipped yet) by listing it in a `releases.yml` on your load path. Release lists
+are unioned by effective date, so adding a release never hides the shipped ones;
+a same-date entry from your file overrides the shipped manifest entry:
+
+```yaml
+# In your application's lib/gov_codes/afsc/releases.yml
+:dafecd:
+- :effective_date: '2026-04-30'
+  :version_label: v3.6
+  :source: DAFECD-30-April-26.pdf
+  :name: Department of the Air Force Enlisted Classification Directory
+```
+
+Pair it with a matching `releases/dafecd/2026-04-30/enlisted.yml` index, and
+`find(code, as_of: "2026-04-30")` will resolve against it.
+
+## Data source & provenance
+
+AFSC data is extracted from the official **Department of the Air Force classification directories** — the DAFECD (enlisted) and DAFOCD (officer) — not third-party sources. Extraction is deterministic, and every code is verified to appear verbatim in the source directory: no predicted or hallucinated codes.
+
+The data is **versioned by each directory's effective date** (the directories are republished roughly semi-annually, on 30 April and 31 October). Look a code up as it stood for a given release, or take the latest:
 
 ```ruby
-# In your application's lib/gov_codes/afsc/enlisted.yml
-9Z:
-  name: Custom AFSC
-  subcategories:
-    0X1:
-      name: Custom Subcategory
-      subcategories:
-        A:
-          name: Custom Shredout
+GovCodes::AFSC.find("1A172Y")                        # latest shipped release
+GovCodes::AFSC.find("1A172Y", as_of: "2025-11-01")   # the release in effect on that date
+GovCodes::AFSC.find("1A172Y").effective_date         # => the release the result came from
 ```
 
-The gem will automatically merge your custom codes with the default codes, overriding any existing codes.
+**Currently shipped:** enlisted AFSCs from the DAFECD effective 31 October 2025. Officer (DAFOCD), reporting/special-duty identifiers, SEIs, prefixes, and Space Force codes are planned.
 
 ## Development
 

bin/extract_afsc_from_pdf.rb

@@ -0,0 +1,182 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Deterministic DAFECD enlisted extractor (Phase C.1a).
+#
+# Reads the official Department of the Air Force Enlisted Classification
+# Directory (DAFECD) PDF, builds a verified specialty-keyed index, writes the
+# versioned release artifact + manifest, and prints a coverage report.
+#
+# Usage:
+#   mise exec -- ruby bin/extract_afsc_from_pdf.rb "DAFECD -31 October 25 v3.5 FINAL.pdf"
+#
+# This is offline dev tooling: it requires pdf-reader (a dev dependency) and is
+# never loaded by the gem runtime.
+
+require "pdf-reader"
+require "yaml"
+require "fileutils"
+require_relative "../lib/gov_codes/dafecd/index_builder"
+require_relative "../lib/gov_codes/dafecd/text"
+require_relative "../lib/gov_codes/dafecd/title_degluer"
+
+DIRECTORY_NAME = "Department of the Air Force Enlisted Classification Directory"
+# The plan estimated ~161 enlisted specialties; the 31 Oct 25 edition actually
+# contains ~136 skill-ladder AFSC specialties (career fields 1-8). Shown for
+# reference only.
+PLAN_ESTIMATE = 161
+
+def effective_date_from(text)
+  raw = text[/DAFECD,\s+(\d{1,2}\s+\w{3,9}\s+\d{2,4})/, 1]
+  raw && GovCodes::Dafecd::Text.parse_date(raw)
+end
+
+def version_label_from(filename)
+  filename[/\bv(\d+(?:\.\d+)?)/i]&.then { |m| "v#{m[/[\d.]+/]}" }
+end
+
+def clean_entry(entry)
+  {
+    name: entry[:name],
+    career_field: entry[:career_field],
+    cem_code: entry[:cem_code],
+    changed_date: entry[:changed_date],
+    skill_levels: entry[:skill_levels].sort.to_h,
+    shredouts: entry[:shredouts].sort_by { |k, _| k.to_s }.to_h
+  }.reject { |_, v| v.nil? }
+end
+
+pdf_file = ARGV[0]
+
+unless pdf_file
+  warn "ERROR: PDF file required"
+  warn "Usage: #{$PROGRAM_NAME} PDF_FILE"
+  exit 1
+end
+
+unless File.exist?(pdf_file)
+  warn "ERROR: File not found: #{pdf_file}"
+  exit 1
+end
+
+puts "=" * 72
+puts "DAFECD enlisted extractor (Phase C.1a)"
+puts "=" * 72
+puts "Source: #{pdf_file}"
+
+full_text = PDF::Reader.new(pdf_file).pages.map(&:text).join("\n")
+puts "Extracted #{full_text.length} characters"
+
+degluer = GovCodes::Dafecd::TitleDegluer.load
+builder = GovCodes::Dafecd::IndexBuilder.new(full_text, degluer: degluer)
+index = builder.build
+
+# --- Fail loudly on any verification failure BEFORE writing ----------------
+# A drifting title override (letters no longer match the source) or an
+# ungrounded code must abort the build rather than emit a stale/hallucinated
+# value.
+if builder.unverified?
+  warn "\nBUILD FAILED: verification gate rejected #{builder.unverified_codes.size} code(s) " \
+       "and #{builder.unverified_titles.size} title override(s)."
+  builder.unverified_codes.uniq.sort.each { |c| warn "  ungrounded code: #{c}" }
+  builder.unverified_titles.each do |t|
+    warn "  drifting title override #{t[:specialty]}: applied=#{t[:applied].inspect} " \
+         "source=#{t[:raw_title].inspect} (#{t[:reason]})"
+  end
+  warn "Nothing was written. Fix the overrides (lib/gov_codes/dafecd/title_overrides.yml)."
+  exit 1
+end
+
+effective_date = effective_date_from(full_text) || "unknown"
+version_label = version_label_from(File.basename(pdf_file))
+
+# --- Write the release artifact -------------------------------------------
+sorted_index = index.sort_by { |k, _| k.to_s }.to_h.transform_values { |e| clean_entry(e) }
+
+release_dir = File.join("lib/gov_codes/afsc/releases/dafecd", effective_date)
+FileUtils.mkdir_p(release_dir)
+enlisted_path = File.join(release_dir, "enlisted.yml")
+
+header = <<~HEADER
+  # DAFECD enlisted AFSC index (specialty-keyed, X-form)
+  # Source: #{File.basename(pdf_file)}
+  # Directory: #{DIRECTORY_NAME}
+  # Effective date: #{effective_date}#{"  (#{version_label})" if version_label}
+  # Generated deterministically by bin/extract_afsc_from_pdf.rb (Phase C.1a).
+  # Do not edit by hand; re-run the extractor against the source PDF.
+HEADER
+
+File.write(enlisted_path, header + sorted_index.to_yaml)
+
+# --- Update the release manifest ------------------------------------------
+manifest_path = "lib/gov_codes/afsc/releases.yml"
+manifest = File.exist?(manifest_path) ? (YAML.safe_load_file(manifest_path, permitted_classes: [Symbol]) || {}) : {}
+manifest[:dafecd] ||= []
+manifest[:dafecd].reject! { |r| r[:effective_date] == effective_date }
+manifest[:dafecd] << {
+  effective_date: effective_date,
+  version_label: version_label,
+  source: File.basename(pdf_file),
+  name: DIRECTORY_NAME
+}
+manifest[:dafecd].sort_by! { |r| r[:effective_date].to_s }
+File.write(manifest_path, manifest.to_yaml)
+
+# --- Coverage report -------------------------------------------------------
+concrete_codes = index.values.flat_map { |e| e[:skill_levels].values.map { |l| l[:code] } }
+cem_codes = index.values.filter_map { |e| e[:cem_code] }
+
+puts
+puts "-" * 72
+puts "COVERAGE REPORT"
+puts "-" * 72
+puts "Effective date:           #{effective_date}"
+puts "Version label:            #{version_label || "(none found)"}"
+puts "(Plan estimated ~#{PLAN_ESTIMATE} specialties; that was high. Actual counts below.)"
+puts
+puts "RECORD RECONCILIATION (split == parsed + merged + dropped)"
+puts "  Records split:          #{builder.records_split}"
+puts "  Specialties parsed:     #{index.size}"
+puts "  Records merged:         #{builder.merged_count}   (duplicate/continuation records folded into an existing specialty)"
+puts "  Records dropped:        #{builder.dropped_records.size}"
+reconciled = index.size + builder.merged_count + builder.dropped_records.size
+puts "  Reconciled total:       #{reconciled}   (#{(reconciled == builder.records_split) ? "OK" : "MISMATCH!"})"
+builder.dropped_records.each do |d|
+  puts "    dropped: cem=#{d[:cem_code].inspect} first=#{d[:first_line].inspect}"
+  puts "             reason: #{d[:reason]}"
+end
+puts
+puts "CODES"
+puts "  Skill-level codes:      #{concrete_codes.size}"
+puts "  CEM codes:              #{cem_codes.size}"
+puts "  Unverified codes:       #{builder.unverified_codes.size}"
+unless builder.unverified_codes.empty?
+  puts "    !! #{builder.unverified_codes.uniq.sort.join(", ")}"
+end
+puts "  NOTE: the code gate is a regression guard — every code here is a verbatim"
+puts "  slice of the source, so 0 unverified is guaranteed by construction."
+puts
+puts "TITLES (de-glued via verified overrides)"
+missing_title = builder.specialties_missing_title
+puts "  Specialties missing title:     #{missing_title.size}"
+puts "    #{missing_title.map(&:to_s).sort.join(", ")}" unless missing_title.empty?
+puts "  Needs de-gluing (no override):  #{builder.specialties_needing_deglue.size}"
+unless builder.specialties_needing_deglue.empty?
+  puts "    #{builder.specialties_needing_deglue.map(&:to_s).sort.join(", ")}"
+end
+puts "  Drifting overrides (rejected):  #{builder.unverified_titles.size}   (build fails if > 0)"
+puts "  Applied clean titles:          #{index.size - builder.specialties_needing_deglue.size}"
+puts "  The title gate is MEANINGFUL: each applied override is verified to match"
+puts "  its raw source title with only spacing/case changed; drift fails the build."
+no_shred = builder.specialties_without_shredouts
+puts "  Specialties without shredouts:  #{no_shred.size}  (normal for many specialties)"
+puts
+puts "SAMPLE DE-GLUED NAMES"
+%i[1A1X2 1C3X1 1Z3X1 2A3X7 3N3X1 4J0X2].each do |spec|
+  puts "  #{spec}: #{index[spec]&.dig(:name).inspect}"
+end
+
+puts
+puts "Wrote #{enlisted_path}"
+puts "Wrote #{manifest_path}"
+puts "Review the diff with 'git diff' before committing (do NOT commit the PDF)."