SF-3817 Partial book drafting

scripts/db_tools/parse-version.ts

@@ -42,7 +42,8 @@ class ParseVersion {
     'Enable Lynx insights',
     'Preview new draft history interface',
     'USFM Format',
-    'Show in-app onboarding form instead of external link'
+    'Show in-app onboarding form instead of external link',
+    'Partial book drafting'
   ];
 
   constructor() {

src/RealtimeServer/scriptureforge/models/translate-config.ts

@@ -74,6 +74,8 @@ export interface DraftConfig {
   draftingSources: TranslateSource[];
   trainingSources: TranslateSource[];
   lastSelectedTrainingDataFiles: string[];
+  /** Training data files available at the last build, to distinguish newly added files from deliberately deselected ones. */
+  lastAvailableTrainingDataFiles?: string[];
   lastSelectedTrainingScriptureRanges?: ProjectScriptureRange[];
   lastSelectedTranslationScriptureRanges?: ProjectScriptureRange[];
   fastTraining?: boolean;

src/RealtimeServer/scriptureforge/services/sf-project-service.ts

@@ -216,6 +216,12 @@ export class SFProjectService extends ProjectService<SFProject> {
                   bsonType: 'string'
                 }
               },
+              lastAvailableTrainingDataFiles: {
+                bsonType: 'array',
+                items: {
+                  bsonType: 'string'
+                }
+              },
               lastSelectedTrainingScriptureRanges: {
                 bsonType: 'array',
                 items: {

src/SIL.XForge.Scripture/ClientApp/e2e/characterize-tests.mts

@@ -6,7 +6,7 @@ import { numberOfTimesToAttemptTest } from './pass-probability.ts';
 import { ScreenshotContext } from './presets.ts';
 import { tests } from './test-definitions.ts';
 
-const retriesToStopAt = 3; // Stop characterization after a test is reliable enough to only need this many retries
+const retriesToStopAt = 4; // Stop characterization after a test is reliable enough to only need this many retries
 const resultFilePath = 'test_characterization.json';
 const testNames = Object.keys(tests) as (keyof typeof tests)[];
 let mostRecentResultData = JSON.parse(await Deno.readTextFile(resultFilePath));

src/SIL.XForge.Scripture/ClientApp/e2e/test-definitions.ts

@@ -6,6 +6,7 @@ import { editTranslation } from './workflows/edit-translation.ts';
 import { generateDraft } from './workflows/generate-draft.ts';
 import { localizedScreenshots } from './workflows/localized-screenshots.ts';
 import { onboardingFlow } from './workflows/onboarding-flow.ts';
+import { partialBookDrafting } from './workflows/partial-book-drafting.ts';
 import { runSmokeTests, traverseHomePageAndLoginPage } from './workflows/smoke-tests.mts';
 
 export const tests = {
@@ -24,6 +25,9 @@ export const tests = {
   generate_draft: async (_engine: BrowserType, page: Page, screenshotContext: ScreenshotContext) => {
     await generateDraft(page, screenshotContext, secrets.users[0]);
   },
+  partial_book_drafting: async (_engine: BrowserType, page: Page, screenshotContext: ScreenshotContext) => {
+    await partialBookDrafting(page, screenshotContext, secrets.users[0]);
+  },
   onboarding_flow: async (engine: BrowserType, page: Page, screenshotContext: ScreenshotContext) => {
     await onboardingFlow(engine, page, screenshotContext, secrets.users[0]);
   },

src/SIL.XForge.Scripture/ClientApp/e2e/test_characterization.json

@@ -26,5 +26,9 @@
   "onboarding_flow": {
     "success": 7,
     "failure": 0
+  },
+  "partial_book_drafting": {
+    "success": 7,
+    "failure": 0
   }
 }

src/SIL.XForge.Scripture/ClientApp/e2e/test_data/partial_draft_training_1.tsv

@@ -0,0 +1,4 @@
+Source	Target
+In the beginning God created the heavens and the earth.	En el principio creó Dios los cielos y la tierra.
+And God said, Let there be light: and there was light.	Y dijo Dios: Sea la luz; y fue la luz.
+And God called the light Day, and the darkness he called Night.	Y llamó Dios a la luz Día, y a las tinieblas llamó Noche.

src/SIL.XForge.Scripture/ClientApp/e2e/test_data/partial_draft_training_2.tsv

@@ -0,0 +1,4 @@
+Source	Target
+Now the serpent was more subtil than any beast of the field.	Pero la serpiente era astuta, más que todos los animales del campo.
+And they heard the voice of the LORD God walking in the garden.	Y oyeron la voz de Jehová Dios que se paseaba en el huerto.
+And the LORD God called unto Adam, and said unto him, Where art thou?	Mas Jehová Dios llamó al hombre, y le dijo: ¿Dónde estás tú?

src/SIL.XForge.Scripture/ClientApp/src/app/app.component.html

@@ -149,6 +149,11 @@
           <button mat-menu-item appRouterLink="/projects" id="project-home-link">
             <mat-icon>home</mat-icon> {{ "my_projects.my_projects" | transloco }}
           </button>
+          @if (experimentalFeatures.showExperimentalFeaturesInMenu) {
+            <button mat-menu-item (click)="openExperimentalFeaturesDialog()">
+              <mat-icon>science</mat-icon> {{ t("experimental_features") }}
+            </button>
+          }
           @if (featureFlags.showDeveloperTools.enabled) {
             <button mat-menu-item [matMenuTriggerFor]="appearanceMenu">
               <mat-icon>palette</mat-icon>

src/SIL.XForge.Scripture/ClientApp/src/app/app.component.ts

@@ -28,6 +28,8 @@ import { DataLoadingComponent } from 'xforge-common/data-loading-component';
 import { DiagnosticOverlayService } from 'xforge-common/diagnostic-overlay.service';
 import { DialogService } from 'xforge-common/dialog.service';
 import { ErrorReportingService } from 'xforge-common/error-reporting.service';
+import { ExperimentalFeaturesDialogComponent } from 'xforge-common/experimental-features/experimental-features-dialog.component';
+import { ExperimentalFeaturesService } from 'xforge-common/experimental-features/experimental-features.service';
 import { ExternalUrlService } from 'xforge-common/external-url.service';
 import { FeatureFlagService } from 'xforge-common/feature-flags/feature-flag.service';
 import { FeatureFlagsDialogComponent } from 'xforge-common/feature-flags/feature-flags-dialog.component';
@@ -138,6 +140,7 @@ export class AppComponent extends DataLoadingComponent implements OnInit, OnDest
     private readonly fontService: FontService,
     private readonly brandingService: BrandingService,
     onlineStatusService: OnlineStatusService,
+    readonly experimentalFeatures: ExperimentalFeaturesService,
     private destroyRef: DestroyRef
   ) {
     super(noticeService, 'AppComponent');
@@ -472,6 +475,10 @@ export class AppComponent extends DataLoadingComponent implements OnInit, OnDest
     this.dialogService.openMatDialog(FeatureFlagsDialogComponent);
   }
 
+  openExperimentalFeaturesDialog(): void {
+    this.dialogService.openMatDialog(ExperimentalFeaturesDialogComponent);
+  }
+
   openDiagnosticOverlay(): void {
     this.diagnosticOverlayService.open();
   }

src/SIL.XForge.Scripture/ClientApp/src/app/app.routes.ts

@@ -31,6 +31,7 @@ import { ConfigureSourcesComponent } from './translate/draft-generation/configur
 import { DraftGenerationComponent } from './translate/draft-generation/draft-generation.component';
 import { DraftOnboardingFormComponent } from './translate/draft-generation/draft-signup-form/draft-onboarding-form.component';
 import { DraftUsfmFormatComponent } from './translate/draft-generation/draft-usfm-format/draft-usfm-format.component';
+import { NewDraftComponent } from './translate/draft-generation/new-draft/new-draft.component';
 import { EditorComponent } from './translate/editor/editor.component';
 import { TranslateOverviewComponent } from './translate/translate-overview/translate-overview.component';
 import { UsersComponent } from './users/users.component';
@@ -82,6 +83,11 @@ export const APP_ROUTES: Routes = [
     component: DraftGenerationComponent,
     canActivate: [NmtDraftAuthGuard]
   },
+  {
+    path: 'projects/:projectId/draft-generation/new-draft',
+    component: NewDraftComponent,
+    canActivate: [NmtDraftAuthGuard]
+  },
   { path: 'projects/:projectId/event-log', component: EventMetricsComponent, canActivate: [EventMetricsAuthGuard] },
   { path: 'projects/:projectId/settings', component: SettingsComponent, canActivate: [SettingsAuthGuard] },
   { path: 'projects/:projectId/sync', component: SyncComponent, canActivate: [SyncAuthGuard] },

src/SIL.XForge.Scripture/ClientApp/src/app/core/permissions.service.ts

@@ -120,6 +120,13 @@ export class PermissionsService {
     return role === SFProjectRole.ParatextAdministrator || role === SFProjectRole.ParatextTranslator;
   }
 
+  /** Whether the user is allowed to configure drafting sources for the project. */
+  canConfigureSources(projectDoc?: SFProjectProfileDoc, userId?: string): boolean {
+    if (projectDoc?.data == null) return false;
+    const role = projectDoc.data.userRoles[userId ?? this.userService.currentUserId];
+    return role === SFProjectRole.ParatextAdministrator;
+  }
+
   canAccessBiblicalTerms(projectDoc: SFProjectProfileDoc): boolean {
     if (projectDoc?.data?.biblicalTermsConfig?.biblicalTermsEnabled !== true) return false;
     return SF_PROJECT_RIGHTS.hasRight(

src/SIL.XForge.Scripture/ClientApp/src/app/core/sf-project.service.ts

@@ -27,7 +27,7 @@ import { QueryParameters, QueryResults } from 'xforge-common/query-parameters';
 import { RealtimeService } from 'xforge-common/realtime.service';
 import { RetryingRequest, RetryingRequestService } from 'xforge-common/retrying-request.service';
 import { EventMetric } from '../event-metrics/event-metric';
-import { BookProgress } from '../shared/progress-service/progress.service';
+import { BookProgressWithChapterProgress } from '../shared/progress-service/progress.service';
 import { expandNumbers } from '../shared/utils';
 import { BiblicalTermDoc } from './models/biblical-term-doc';
 import { InviteeStatus } from './models/invitee-status';
@@ -98,8 +98,10 @@ export class SFProjectService extends ProjectService<SFProject, SFProjectDoc> {
       // If the book is present without chapters, then the chapter is present
       if (range.length === bookId.length) return true;
 
-      // Expand the chapter range, and see if the specified chapter number is present
-      if (expandNumbers(range.slice(bookId.length)).includes(chapterNum)) return true;
+      // Expand the chapter range, and see if the specified chapter number is present. Invalid notation expands to
+      // null and is treated as the chapter not being present.
+      const chapters: number[] | null = expandNumbers(range.slice(bookId.length));
+      if (chapters != null && chapters.includes(chapterNum)) return true;
     }
 
     // The chapter was not present in the book
@@ -451,7 +453,7 @@ export class SFProjectService extends ProjectService<SFProject, SFProjectDoc> {
   }
 
   /** Gets project progress by calling the backend aggregation endpoint. */
-  async getProjectProgress(projectId: string): Promise<BookProgress[]> {
-    return await this.onlineInvoke<BookProgress[]>('getProjectProgress', { projectId });
+  async getProjectProgress(projectId: string): Promise<BookProgressWithChapterProgress[]> {
+    return await this.onlineInvoke<BookProgressWithChapterProgress[]>('getProjectProgress', { projectId });
   }
 }

src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/draft-jobs.component.html

@@ -128,9 +128,9 @@
                 }}</a
                 >:
                 @if (projectBook.books.length <= 2) {
-                  {{ projectBook.books.join("; ") }}
+                  {{ projectBook.booksDisplay }}
                 } @else {
-                  <span [matTooltip]="projectBook.books.join('; ')" class="book-count">
+                  <span [matTooltip]="projectBook.booksDisplay" class="book-count">
                     {{ projectBook.books.length }} books
                   </span>
                 }
@@ -150,9 +150,9 @@
                 }}</a
                 >:
                 @if (projectBook.books.length <= 2) {
-                  {{ projectBook.books.join("; ") }}
+                  {{ projectBook.booksDisplay }}
                 } @else {
-                  <span [matTooltip]="projectBook.books.join('; ')" class="book-count">
+                  <span [matTooltip]="projectBook.booksDisplay" class="book-count">
                     {{ projectBook.books.length }} books
                   </span>
                 }

src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/draft-jobs.component.ts

@@ -34,6 +34,8 @@ import { SFProjectService } from '../core/sf-project.service';
 import { EventMetric } from '../event-metrics/event-metric';
 import { InfoComponent } from '../shared/info/info.component';
 import { NoticeComponent } from '../shared/notice/notice.component';
+import { trainingSourceRangesWithTargetDetail } from '../shared/scripture-range';
+import { formatScriptureRangeTokensCompact } from '../shared/scripture-range-display';
 import { projectLabel } from '../shared/utils';
 import { DateRangePickerComponent, NormalizedDateRange } from './date-range-picker.component';
 import { DraftJobsExportService, SpreadsheetRow } from './draft-jobs-export.service';
@@ -94,6 +96,8 @@ export interface DraftJobsProjectBooks {
   /** Short name of the project if available. */
   shortName?: string;
   books: string[];
+  /** Compact display of `books`, e.g. "GEN-LEV; NUM 1-3" (full books collapse into ranges). */
+  booksDisplay: string;
 }
 
 /** Names of events that are relevant for pre-translation draft generation requests and processing. */
@@ -829,20 +833,22 @@ export class DraftJobsComponent extends DataLoadingComponent implements OnInit {
       if (event.payload != null) {
         const buildConfig = event.payload.buildConfig;
         if (buildConfig != null) {
-          // Extract training books
+          // Extract training books. The entry for the event's own project (the draft target) is not a training
+          // source; it is folded into the source ranges as chapter detail.
           if (Array.isArray(buildConfig.TrainingScriptureRanges)) {
-            for (const range of buildConfig.TrainingScriptureRanges) {
-              if (range.ScriptureRange != null) {
-                // Use the project ID from the range if available, otherwise use the event's project ID
-                const projectId = range.ProjectId || event.projectId;
-                if (projectId) {
-                  // Split semicolon-separated books and add them to the project's books
-                  const books = range.ScriptureRange.split(';').filter((book: string) => book.trim().length > 0);
-                  if (!trainingProjects.has(projectId)) {
-                    trainingProjects.set(projectId, []);
-                  }
-                  trainingProjects.get(projectId)!.push(...books);
+            const ranges: { projectId?: string; scriptureRange?: string }[] = buildConfig.TrainingScriptureRanges.map(
+              (range: any) => ({ projectId: range.ProjectId, scriptureRange: range.ScriptureRange ?? undefined })
+            );
+            for (const range of trainingSourceRangesWithTargetDetail(ranges, r => r.projectId, event.projectId)) {
+              // Use the project ID from the range if available, otherwise use the event's project ID
+              const projectId = range.projectId || event.projectId;
+              if (projectId) {
+                // Split semicolon-separated books and add them to the project's books
+                const books = range.scriptureRange!.split(';').filter((book: string) => book.trim().length > 0);
+                if (!trainingProjects.has(projectId)) {
+                  trainingProjects.set(projectId, []);
                 }
+                trainingProjects.get(projectId)!.push(...books);
               }
             }
           }
@@ -874,14 +880,16 @@ export class DraftJobsComponent extends DataLoadingComponent implements OnInit {
     const trainingBooks: DraftJobsProjectBooks[] = Array.from(trainingProjects.entries()).map(([projectId, books]) => ({
       sfProjectId: projectId,
       projectDisplayName: '',
-      books: books
+      books: books,
+      booksDisplay: formatScriptureRangeTokensCompact(books)
     }));
 
     const translationBooks: DraftJobsProjectBooks[] = Array.from(translationProjects.entries()).map(
       ([projectId, books]) => ({
         sfProjectId: projectId,
         projectDisplayName: '',
-        books: books
+        books: books,
+        booksDisplay: formatScriptureRangeTokensCompact(books)
       })
     );
 

src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-build-report.ts

@@ -214,8 +214,9 @@ function parseBooksAndChapters(scriptureRange: string): BookAndChapters[] {
       if (chapterPart === '') {
         return { bookId };
       }
-      const chapters: number[] = expandNumbers(chapterPart);
-      return chapters.length > 0 ? { bookId, chapters } : { bookId };
+      // Invalid chapter notation falls back to the whole book (no chapter detail), same as an empty chapter part.
+      const chapters: number[] | null = expandNumbers(chapterPart);
+      return chapters != null && chapters.length > 0 ? { bookId, chapters } : { bookId };
     })
     .filter(notNull);
 }

src/SIL.XForge.Scripture/ClientApp/src/app/serval-administration/serval-builds.component.spec.ts

@@ -67,7 +67,7 @@
       { provide: UserService, useMock: mockUserService },
       { provide: I18nService, useMock: mockI18nService },
       { provide: ActivatedRoute, useMock: mockedActivatedRoute },
       provideNoopAnimations()
     ]
   }));
 
@@ -1726,11 +1726,43 @@
       expect(result).toBe('112233 BSB: GEN 10-11, 16-19; EXO');
     });
 
-    it('compactRangeNotation de-duplicates duplicate chapter numbers', () => {
+    it('collapses contiguous full books into an ID range', () => {
+      const projectBooks: ProjectBooks[] = [
+        {
+          sfProjectId: '112233',
+          projectDisplayName: 'BSB',
+          shortName: 'BSB',
+          booksAndChapters: [
+            { bookId: 'GEN' },
+            { bookId: 'EXO' },
+            { bookId: 'LEV' },
+            { bookId: 'NUM', chapters: [1, 2, 3] }
+          ]
+        }
+      ];
+
+      // SUT
+      const result: string = ServalBuildsComponent.formatProjectBooks(projectBooks);
+
+      expect(result).toBe('112233 BSB: GEN-LEV; NUM 1-3');
+    });
+
+    it('treats a book whose chapters reach the canonical count as a full book', () => {
+      const projectBooks: ProjectBooks[] = [
+        {
+          sfProjectId: '112233',
+          projectDisplayName: 'BSB',
+          shortName: 'BSB',
+          booksAndChapters: [{ bookId: 'GEN', chapters: [1, 2, 3] }, { bookId: 'EXO' }]
+        }
+      ];
+      // All 40 chapters of Exodus, which is the same as the whole book
+      projectBooks[0].booksAndChapters[1].chapters = Array.from({ length: 40 }, (_, i) => i + 1);
+
       // SUT
-      const result: string = ServalBuildsComponent.compactRangeNotation([10, 10, 11, 10, 16, 16, 17]);
+      const result: string = ServalBuildsComponent.formatProjectBooks(projectBooks);
 
-      expect(result).toBe('10-11, 16-17');
+      expect(result).toBe('112233 BSB: GEN 1-3; EXO');
     });
   });