From 2c52e03fd15b82e2c8ec4479f01ecf3b7bac17a8 Mon Sep 17 00:00:00 2001 From: hissy Date: Sun, 19 Jul 2026 22:04:56 +0900 Subject: [PATCH 1/3] Add URL inventory dashboard and file URL normalization. Track import URLs from CSV, assign them to batches, sync status from logs, and resolve relative file URLs in transformers. Co-authored-by: Cursor --- controller.php | 2 +- controllers/element/dashboard/urls/header.php | 13 + .../system/content_importer/batches.php | 5 + .../system/content_importer/urls.php | 549 ++++++++++++++++++ .../transformer/normalize_file_url.php | 17 + elements/dashboard/urls/header.php | 9 + .../system/content_importer/urls.php | 239 ++++++++ .../content_importer/urls/import_csv.php | 23 + .../system/content_importer/urls/map_csv.php | 92 +++ .../urls/set_imported_page.php | 39 ++ src/Command/ImportUrlCsvChunkCommand.php | 29 + .../ImportUrlCsvChunkCommandHandler.php | 17 + src/Csv/UrlCsvImporter.php | 194 +++++++ src/Entity/ImportUrl.php | 270 +++++++++ src/Install/Installer.php | 1 + src/Publisher/BatchPublisher.php | 5 + src/Repository/ImportUrlRepository.php | 41 ++ src/Search/ImportUrlList.php | 93 +++ src/Service/ImportUrlStatusSync.php | 91 +++ src/Service/UrlBatchAssigner.php | 161 +++++ src/Traits/FileImporterTrait.php | 70 +++ src/Traits/NormalizeFileUrlTrait.php | 203 +++++++ .../NormalizeFileUrlAttributeTransformer.php | 54 ++ .../NormalizeFileUrlContentTransformer.php | 80 +++ .../TransformerServiceProvider.php | 2 + 25 files changed, 2298 insertions(+), 1 deletion(-) create mode 100644 controllers/element/dashboard/urls/header.php create mode 100644 controllers/single_page/dashboard/system/content_importer/urls.php create mode 100644 elements/content_importer/transformer/normalize_file_url.php create mode 100644 elements/dashboard/urls/header.php create mode 100644 single_pages/dashboard/system/content_importer/urls.php create mode 100644 single_pages/dashboard/system/content_importer/urls/import_csv.php create mode 100644 single_pages/dashboard/system/content_importer/urls/map_csv.php create mode 100644 single_pages/dashboard/system/content_importer/urls/set_imported_page.php create mode 100644 src/Command/ImportUrlCsvChunkCommand.php create mode 100644 src/Command/ImportUrlCsvChunkCommandHandler.php create mode 100644 src/Csv/UrlCsvImporter.php create mode 100644 src/Entity/ImportUrl.php create mode 100644 src/Repository/ImportUrlRepository.php create mode 100644 src/Search/ImportUrlList.php create mode 100644 src/Service/ImportUrlStatusSync.php create mode 100644 src/Service/UrlBatchAssigner.php create mode 100644 src/Traits/NormalizeFileUrlTrait.php create mode 100644 src/Transformer/NormalizeFileUrlAttributeTransformer.php create mode 100644 src/Transformer/NormalizeFileUrlContentTransformer.php diff --git a/controller.php b/controller.php index a6c657d..0e45e6a 100644 --- a/controller.php +++ b/controller.php @@ -17,7 +17,7 @@ class Controller extends Package protected $pkgHandle = 'md_content_importer'; - protected $pkgVersion = '1.1.1'; + protected $pkgVersion = '1.2.0'; protected $pkgAutoloaderRegistries = [ 'src' => '\Macareux\ContentImporter', diff --git a/controllers/element/dashboard/urls/header.php b/controllers/element/dashboard/urls/header.php new file mode 100644 index 0000000..1179b30 --- /dev/null +++ b/controllers/element/dashboard/urls/header.php @@ -0,0 +1,13 @@ +entityManager->persist($batch); $this->entityManager->flush(); + /** @var UrlBatchAssigner $assigner */ + $assigner = $this->app->make(UrlBatchAssigner::class); + $assigner->syncFromBatchSourcePath($batch); + $this->flash('success', t('Batch saved successfully.')); if ($batchID) { diff --git a/controllers/single_page/dashboard/system/content_importer/urls.php b/controllers/single_page/dashboard/system/content_importer/urls.php new file mode 100644 index 0000000..8d6563c --- /dev/null +++ b/controllers/single_page/dashboard/system/content_importer/urls.php @@ -0,0 +1,549 @@ +app->make('session'); + } + + public function view() + { + /** @var ImportUrlList $list */ + $list = $this->app->make(ImportUrlList::class); + $list->setItemsPerPage(50); + + $sort = (string) $this->request->query->get('sort', 'url'); + $sortColumns = [ + 'id' => 'u.id', + 'url' => 'u.url', + 'import_date' => 'u.importDate', + ]; + if (!isset($sortColumns[$sort])) { + $sort = 'url'; + } + + $direction = strtolower((string) $this->request->query->get('direction', 'asc')); + if (!in_array($direction, ['asc', 'desc'], true)) { + $direction = 'asc'; + } + $list->sortBy($sortColumns[$sort], strtoupper($direction)); + + $keywords = trim((string) $this->request->query->get('keywords', '')); + if ($keywords !== '') { + $list->filterByKeywords($keywords); + } + + $status = (string) $this->request->query->get('status', ''); + if ($status !== '' && array_key_exists($status, ImportUrl::getStatusOptions())) { + $list->filterByStatus($status); + } + + $batchId = (int) $this->request->query->get('batch_id', 0); + if ($batchId > 0) { + $list->filterByBatchId($batchId); + } + + $factory = new PaginationFactory(Request::getInstance()); + $pagination = $factory->createPaginationObject($list, PaginationFactory::PERMISSIONED_PAGINATION_STYLE_PAGER); + + $batchOptions = ['' => t('** All Batches')]; + $assignBatchOptions = ['' => t('** Select Batch')]; + foreach ($this->getAll(Batch::class) as $batch) { + /** @var Batch $batch */ + $batchOptions[$batch->getId()] = $batch->getName(); + $assignBatchOptions[$batch->getId()] = $batch->getName(); + } + + $this->set('list', $list); + $this->set('pagination', $pagination); + $this->set('keywords', $keywords); + $this->set('status', $status); + $this->set('batchId', $batchId); + $this->set('sort', $sort); + $this->set('direction', $direction); + $this->set('statusOptions', ['' => t('** All Statuses')] + ImportUrl::getStatusOptions()); + $this->set('batchOptions', $batchOptions); + $this->set('assignBatchOptions', $assignBatchOptions); + $this->set('token', $this->token); + $this->set('headerMenu', $this->app->make(ElementManager::class)->get('dashboard/urls/header', 'md_content_importer')); + } + + public function import_csv() + { + $this->set('pageTitle', t('Import URL CSV')); + $this->set('token', $this->token); + $this->render('/dashboard/system/content_importer/urls/import_csv'); + } + + public function upload_csv() + { + if (!$this->token->validate('upload_csv')) { + $this->error->add($this->token->getErrorMessage()); + } + + /** @var UploadedFile|null $file */ + $file = $this->request->files->get('csv_file'); + if (!$file instanceof UploadedFile || !$file->isValid()) { + $this->error->add(t('Please upload a CSV file.')); + } else { + $extension = strtolower($file->getClientOriginalExtension()); + if ($extension !== 'csv') { + $this->error->add(t('The uploaded file must be a CSV file.')); + } + } + + if (!$this->error->has()) { + /** @var \Concrete\Core\File\Service\File $fileService */ + $fileService = $this->app->make('helper/file'); + $tmpDir = $fileService->getTemporaryDirectory(); + $safeName = 'md_ci_urls_' . bin2hex(random_bytes(16)) . '.csv'; + $targetPath = rtrim($tmpDir, DIRECTORY_SEPARATOR) . DIRECTORY_SEPARATOR . $safeName; + + $realTmp = realpath($tmpDir); + if ($realTmp === false) { + $this->error->add(t('Unable to resolve temporary directory.')); + } else { + $file->move($tmpDir, $safeName); + $realPath = realpath($targetPath); + if ($realPath === false || strpos($realPath, $realTmp) !== 0) { + $this->error->add(t('Invalid upload path.')); + } else { + try { + /** @var UrlCsvImporter $importer */ + $importer = $this->app->make(UrlCsvImporter::class); + $headers = $importer->getHeaders($realPath); + if ($headers === []) { + $this->error->add(t('CSV file has no header row.')); + @unlink($realPath); + } else { + $this->getCsvSession()->set(self::SESSION_CSV_KEY, [ + 'path' => $realPath, + 'headers' => $headers, + ]); + + return $this->buildRedirect($this->action('map_csv')); + } + } catch (\Throwable $e) { + $this->error->add(t('Unable to read CSV file: %s', $e->getMessage())); + @unlink($targetPath); + } + } + } + } + + $this->import_csv(); + } + + public function map_csv() + { + $state = $this->getCsvSession()->get(self::SESSION_CSV_KEY); + if (!is_array($state) || empty($state['path']) || empty($state['headers'])) { + $this->error->add(t('Please upload a CSV file first.')); + $this->import_csv(); + + return; + } + + $path = (string) $state['path']; + $realPath = realpath($path); + if ($realPath === false || !is_file($realPath)) { + $this->getCsvSession()->remove(self::SESSION_CSV_KEY); + $this->error->add(t('Uploaded CSV file is no longer available. Please upload again.')); + $this->import_csv(); + + return; + } + + $headers = $state['headers']; + $headerOptions = []; + foreach ($headers as $header) { + $headerOptions[$header] = $header; + } + + $this->set('pageTitle', t('Map CSV Columns')); + $this->set('token', $this->token); + $this->set('headers', $headers); + $this->set('headerOptions', $headerOptions); + $this->set('urlColumn', UrlCsvImporter::suggestUrlColumn($headers) ?? ''); + $this->set('titleColumn', UrlCsvImporter::suggestTitleColumn($headers) ?? ''); + $this->set('defaultFilters', self::defaultFiltersForHeaders($headers)); + $this->set('operatorOptions', [ + 'equals' => t('Equals'), + 'contains' => t('Contains'), + ]); + $this->render('/dashboard/system/content_importer/urls/map_csv'); + } + + public function submit_csv_import() + { + if (!$this->token->validate('submit_csv_import')) { + $this->error->add($this->token->getErrorMessage()); + } + + $state = $this->getCsvSession()->get(self::SESSION_CSV_KEY); + if (!is_array($state) || empty($state['path']) || empty($state['headers'])) { + $this->error->add(t('Please upload a CSV file first.')); + } + + $urlColumn = (string) $this->post('urlColumn'); + $titleColumn = (string) $this->post('titleColumn'); + $headers = is_array($state['headers'] ?? null) ? $state['headers'] : []; + + if ($urlColumn === '' || !in_array($urlColumn, $headers, true)) { + $this->error->add(t('Please select a valid URL column.')); + } + if ($titleColumn !== '' && !in_array($titleColumn, $headers, true)) { + $this->error->add(t('Please select a valid title column.')); + } + + $filters = $this->parseFilters($headers); + $path = realpath((string) ($state['path'] ?? '')); + if ($path === false || !is_file($path)) { + $this->error->add(t('Uploaded CSV file is no longer available. Please upload again.')); + } + + if (!$this->error->has()) { + try { + /** @var UrlCsvImporter $importer */ + $importer = $this->app->make(UrlCsvImporter::class); + $rows = $importer->getMatchingRows( + $path, + $urlColumn, + $titleColumn !== '' ? $titleColumn : null, + $filters + ); + + if ($rows === []) { + $this->error->add(t('No rows matched the selected filters.')); + } else { + $chunks = array_chunk($rows, UrlCsvImporter::CHUNK_SIZE); + $commandBatch = \Concrete\Core\Command\Batch\Batch::create(t('Import URLs'), function () use ($chunks) { + foreach ($chunks as $chunk) { + $command = new ImportUrlCsvChunkCommand(); + $command->setRows($chunk); + yield $command; + } + }); + + $this->getCsvSession()->remove(self::SESSION_CSV_KEY); + @unlink($path); + + return $this->dispatchBatch($commandBatch); + } + } catch (\Throwable $e) { + $this->error->add(t('Unable to import CSV: %s', $e->getMessage())); + } + } + + $this->map_csv(); + } + + public function import_completed() + { + $this->flash('success', t('CSV import completed.')); + + return $this->buildRedirect($this->action('view')); + } + + public function assign_to_batch() + { + if (!$this->token->validate('assign_to_batch')) { + $this->error->add($this->token->getErrorMessage()); + } + + $batchId = (int) $this->post('assign_batch_id'); + /** @var Batch|null $batch */ + $batch = $this->getEntry(Batch::class, $batchId); + if (!$batch) { + $this->error->add(t('Please select a valid batch.')); + } + + $urls = $this->getSelectedImportUrls(); + if ($urls === []) { + $this->error->add(t('Please select at least one URL.')); + } + + if (!$this->error->has()) { + /** @var UrlBatchAssigner $assigner */ + $assigner = $this->app->make(UrlBatchAssigner::class); + $errors = []; + $assigned = $assigner->assignUrlsToBatch($urls, $batch, $errors); + foreach ($errors as $message) { + $this->error->add($message); + } + + if ($assigned > 0) { + $this->flash('success', t2('%s URL assigned to batch.', '%s URLs assigned to batch.', $assigned)); + } + + if (!$this->error->has()) { + return $this->buildRedirect($this->action('view')); + } + } + + $this->view(); + } + + public function unassign() + { + if (!$this->token->validate('unassign')) { + $this->error->add($this->token->getErrorMessage()); + } + + $urls = $this->getSelectedImportUrls(); + if ($urls === []) { + $this->error->add(t('Please select at least one URL.')); + } + + if (!$this->error->has()) { + /** @var UrlBatchAssigner $assigner */ + $assigner = $this->app->make(UrlBatchAssigner::class); + $unassigned = $assigner->unassignUrls($urls); + $this->flash('success', t2('%s URL unassigned.', '%s URLs unassigned.', $unassigned)); + + return $this->buildRedirect($this->action('view')); + } + + $this->view(); + } + + public function remove_urls() + { + if (!$this->token->validate('remove_urls')) { + $this->error->add($this->token->getErrorMessage()); + } + + $urls = $this->getSelectedImportUrls(); + if ($urls === []) { + $this->error->add(t('Please select at least one URL.')); + } + + if (!$this->error->has()) { + /** @var UrlBatchAssigner $assigner */ + $assigner = $this->app->make(UrlBatchAssigner::class); + $assigner->unassignUrls($urls); + + foreach ($urls as $importUrl) { + $this->entityManager->remove($importUrl); + } + $this->entityManager->flush(); + + $removed = count($urls); + $this->flash('success', t2('%s URL removed.', '%s URLs removed.', $removed)); + + return $this->buildRedirect($this->action('view')); + } + + $this->view(); + } + + public function set_imported_page($id) + { + /** @var ImportUrl|null $importUrl */ + $importUrl = $this->getEntry(ImportUrl::class, (int) $id); + if (!$importUrl) { + $this->error->add(t('Invalid URL.')); + $this->view(); + + return; + } + + $this->set('importUrl', $importUrl); + $this->set('pageSelector', $this->app->make('helper/form/page_selector')); + $this->set('token', $this->token); + $this->set('pageTitle', t('Set Imported Page')); + $this->render('/dashboard/system/content_importer/urls/set_imported_page'); + } + + public function submit_imported_page() + { + if (!$this->token->validate('submit_imported_page')) { + $this->error->add($this->token->getErrorMessage()); + } + + /** @var ImportUrl|null $importUrl */ + $importUrl = $this->getEntry(ImportUrl::class, (int) $this->post('url_id')); + if (!$importUrl) { + $this->error->add(t('Invalid URL.')); + } + + $page = Page::getByID((int) $this->post('imported_cID')); + if (!$page || $page->isError()) { + $this->error->add(t('Please select a valid imported page.')); + } else { + $permissions = new Checker($page); + if (!$permissions->canViewPage()) { + $this->error->add(t('You do not have permission to view the selected page.')); + } + } + + if (!$this->error->has()) { + if ($importUrl->getBatch()) { + /** @var UrlBatchAssigner $assigner */ + $assigner = $this->app->make(UrlBatchAssigner::class); + $assigner->unassignUrls([$importUrl]); + } + + $importUrl->setImportedPage($page); + $importUrl->setImportDate(CarbonImmutable::now()); + $importUrl->setManuallyImported(true); + $this->entityManager->persist($importUrl); + $this->entityManager->flush(); + + $this->flash('success', t('Imported page set successfully.')); + + return $this->buildRedirect($this->action('view')); + } + + if ($importUrl) { + $this->set_imported_page($importUrl->getId()); + } else { + $this->view(); + } + } + + public function refresh_status() + { + if (!$this->token->validate('refresh_status')) { + $this->error->add($this->token->getErrorMessage()); + $this->view(); + + return; + } + + /** @var ImportUrlStatusSync $sync */ + $sync = $this->app->make(ImportUrlStatusSync::class); + $updated = $sync->refreshFromLogs(); + $this->flash('success', t2('%s URL updated from import logs.', '%s URLs updated from import logs.', $updated)); + + return $this->buildRedirect($this->action('view')); + } + + /** + * @return ImportUrl[] + */ + protected function getSelectedImportUrls(): array + { + $ids = $this->post('url_ids'); + if (!is_array($ids)) { + return []; + } + + /** @var Numbers $valn */ + $valn = $this->app->make('helper/validation/numbers'); + /** @var EntityManagerInterface $em */ + $em = $this->app->make(EntityManagerInterface::class); + $urls = []; + + foreach ($ids as $id) { + if (!$valn->integer($id, 1)) { + continue; + } + $importUrl = $em->find(ImportUrl::class, (int) $id); + if ($importUrl) { + $urls[] = $importUrl; + } + } + + return $urls; + } + + /** + * @param string[] $headers + * + * @return array{column: string, operator: string, value: string}[] + */ + protected static function defaultFiltersForHeaders(array $headers): array + { + $filters = []; + if (in_array('Content Type', $headers, true)) { + $filters[] = [ + 'column' => 'Content Type', + 'operator' => 'equals', + 'value' => 'text/html; charset=UTF-8', + ]; + } + if (in_array('Status Code', $headers, true)) { + $filters[] = [ + 'column' => 'Status Code', + 'operator' => 'equals', + 'value' => '200', + ]; + } + if ($filters === []) { + $filters[] = [ + 'column' => '', + 'operator' => 'equals', + 'value' => '', + ]; + } + + return $filters; + } + + /** + * @param string[] $headers + * + * @return array{column: string, operator: string, value: string}[] + */ + protected function parseFilters(array $headers): array + { + $columns = $this->post('filter_column'); + $operators = $this->post('filter_operator'); + $values = $this->post('filter_value'); + if (!is_array($columns)) { + return []; + } + + $filters = []; + foreach ($columns as $index => $column) { + $column = (string) $column; + if ($column === '' || !in_array($column, $headers, true)) { + continue; + } + $operator = (string) ($operators[$index] ?? 'equals'); + if (!in_array($operator, ['equals', 'contains'], true)) { + $operator = 'equals'; + } + $value = (string) ($values[$index] ?? ''); + if ($value === '') { + continue; + } + $filters[] = [ + 'column' => $column, + 'operator' => $operator, + 'value' => $value, + ]; + } + + return $filters; + } +} diff --git a/elements/content_importer/transformer/normalize_file_url.php b/elements/content_importer/transformer/normalize_file_url.php new file mode 100644 index 0000000..2735e97 --- /dev/null +++ b/elements/content_importer/transformer/normalize_file_url.php @@ -0,0 +1,17 @@ + +
+ label('documentRoot', t('Document Root')) ?> + text('documentRoot', $documentRoot, [ + 'placeholder' => 'https://www.example.com/path/', + 'aria-describedby' => 'normalizeDocumentRootHelp', + ]) ?> +
+ ../') ?> +
+
diff --git a/elements/dashboard/urls/header.php b/elements/dashboard/urls/header.php new file mode 100644 index 0000000..28d0d03 --- /dev/null +++ b/elements/dashboard/urls/header.php @@ -0,0 +1,9 @@ + +
+ +
diff --git a/single_pages/dashboard/system/content_importer/urls.php b/single_pages/dashboard/system/content_importer/urls.php new file mode 100644 index 0000000..4d4dc0a --- /dev/null +++ b/single_pages/dashboard/system/content_importer/urls.php @@ -0,0 +1,239 @@ + $keywords, + 'status' => $status, + 'batch_id' => $batchId ?: null, + 'sort' => $column, + 'direction' => $nextDirection, + ], static function ($value): bool { + return $value !== null && $value !== ''; + }); + + return (string) $view->action('view') . '?' . http_build_query($query); +}; +$sortIndicator = static function (string $column) use ($sort, $direction): string { + if ($sort !== $column) { + return ''; + } + + return $direction === 'asc' ? ' ↑' : ' ↓'; +}; +?> +
+ hidden('sort', $sort) ?> + hidden('direction', $direction) ?> +
+
+ label('keywords', t('URL')) ?> + text('keywords', $keywords) ?> +
+
+ label('status', t('Status')) ?> + select('status', $statusOptions, $status) ?> +
+
+ label('batch_id', t('Batch')) ?> + select('batch_id', $batchOptions, $batchId) ?> +
+
+ +
+
+
+ +
+ output('refresh_status') ?> + +
+ + +
+ select('assign_batch_id_ui', $assignBatchOptions, '', ['class' => 'form-select', 'style' => 'max-width: 280px', 'id' => 'ccm-assign-batch-id']) ?> + + + +
+ +
+ + + + + + + + + + + + + getCurrentPageResults() as $importUrl) { + $batch = $importUrl->getBatch(); + $page = $importUrl->getImportedPage(); + ?> + + + + + + + + + + +
+ + + + + + + +
+ + + getUrl()) ?> + getTitle()) { ?> +
getTitle()) ?>
+ + 		getStatusLabel()) ?> + + + getName()) ?> + + isManuallyImported()) { ?> + + + — + + + + + getCollectionPath() ?: $page->getCollectionName()) ?> + +
+ + + +
+ getImportedCID()) { ?> + +
+ + + +
+ + + + + + 		+ getImportDate()) { + echo h($importUrl->getImportDate()->format('Y-m-d H:i:s')); + } else { + echo '—'; + } + ?> +
+
+ renderView('dashboard') ?> + + + + + + + diff --git a/single_pages/dashboard/system/content_importer/urls/import_csv.php b/single_pages/dashboard/system/content_importer/urls/import_csv.php new file mode 100644 index 0000000..370a5a5 --- /dev/null +++ b/single_pages/dashboard/system/content_importer/urls/import_csv.php @@ -0,0 +1,23 @@ + +
+ output('upload_csv') ?> +
+ label('csv_file', t('CSV File')) ?> + file('csv_file', ['accept' => '.csv,text/csv']) ?> +
+ +
+
+
+
+ + +
+
+
diff --git a/single_pages/dashboard/system/content_importer/urls/map_csv.php b/single_pages/dashboard/system/content_importer/urls/map_csv.php new file mode 100644 index 0000000..611ed5d --- /dev/null +++ b/single_pages/dashboard/system/content_importer/urls/map_csv.php @@ -0,0 +1,92 @@ + +
+ output('submit_csv_import') ?> + +
+ +
+ label('urlColumn', t('URL Column')) ?> + select('urlColumn', $headerOptions, $urlColumn) ?> +
+
+ label('titleColumn', t('Title Column')) ?> + select('titleColumn', ['' => t('** None')] + $headerOptions, $titleColumn) ?> +
+
+ +
+ +

+
+ +
+
+ select('filter_column[]', ['' => t('** Column')] + $headerOptions, $filter['column']) ?> +
+
+ select('filter_operator[]', $operatorOptions, $filter['operator']) ?> +
+
+ text('filter_value[]', $filter['value']) ?> +
+
+ +
+
+ +
+ +
+ +
+
+ + +
+
+
+ + diff --git a/single_pages/dashboard/system/content_importer/urls/set_imported_page.php b/single_pages/dashboard/system/content_importer/urls/set_imported_page.php new file mode 100644 index 0000000..86a7a20 --- /dev/null +++ b/single_pages/dashboard/system/content_importer/urls/set_imported_page.php @@ -0,0 +1,39 @@ +getImportedPage(); +?> +
+ output('submit_imported_page') ?> + hidden('url_id', $importUrl->getId()) ?> + +
+ label('original_url', t('Original URL')) ?> +
+ + getUrl()) ?> + +
+
+ +
+ label('imported_cID', t('Imported Page')) ?> + selectPage('imported_cID', $importedPage ? $importedPage->getCollectionID() : null) ?> +
+ +
+
+ +
+
+ + +
+
+
diff --git a/src/Command/ImportUrlCsvChunkCommand.php b/src/Command/ImportUrlCsvChunkCommand.php new file mode 100644 index 0000000..12ee532 --- /dev/null +++ b/src/Command/ImportUrlCsvChunkCommand.php @@ -0,0 +1,29 @@ +rows; + } + + /** + * @param array{url: string, title: ?string, metadata: array}[] $rows + */ + public function setRows(array $rows): void + { + $this->rows = $rows; + } +} diff --git a/src/Command/ImportUrlCsvChunkCommandHandler.php b/src/Command/ImportUrlCsvChunkCommandHandler.php new file mode 100644 index 0000000..ed63323 --- /dev/null +++ b/src/Command/ImportUrlCsvChunkCommandHandler.php @@ -0,0 +1,17 @@ +make(UrlCsvImporter::class); + $importer->upsertRows($command->getRows()); + } +} diff --git a/src/Csv/UrlCsvImporter.php b/src/Csv/UrlCsvImporter.php new file mode 100644 index 0000000..e7e8334 --- /dev/null +++ b/src/Csv/UrlCsvImporter.php @@ -0,0 +1,194 @@ +entityManager = $entityManager; + } + + /** + * @return string[] + */ + public function getHeaders(string $filePath): array + { + $reader = $this->createReader($filePath); + + return $reader->getHeader(); + } + + /** + * @param array{column: string, operator: string, value: string}[] $filters + * + * @return array{url: string, title: ?string, metadata: array}[] + */ + public function getMatchingRows(string $filePath, string $urlColumn, ?string $titleColumn, array $filters = []): array + { + $reader = $this->createReader($filePath); + $headers = $reader->getHeader(); + if (!in_array($urlColumn, $headers, true)) { + throw new \InvalidArgumentException(t('Invalid URL column.')); + } + if ($titleColumn !== null && $titleColumn !== '' && !in_array($titleColumn, $headers, true)) { + throw new \InvalidArgumentException(t('Invalid title column.')); + } + + $rows = []; + foreach ($reader->getRecords() as $record) { + if (!$this->matchesFilters($record, $filters)) { + continue; + } + + $url = trim((string) ($record[$urlColumn] ?? '')); + if ($url === '' || filter_var($url, FILTER_VALIDATE_URL) === false) { + continue; + } + + $title = null; + if ($titleColumn !== null && $titleColumn !== '') { + $titleValue = trim((string) ($record[$titleColumn] ?? '')); + if ($titleValue !== '') { + $title = mb_substr($titleValue, 0, 255); + } + } + + $metadata = []; + foreach (['Content Type', 'Status Code', 'Status', 'Indexability'] as $metaColumn) { + if (array_key_exists($metaColumn, $record) && $record[$metaColumn] !== null && $record[$metaColumn] !== '') { + $metadata[$metaColumn] = (string) $record[$metaColumn]; + } + } + + $rows[] = [ + 'url' => $url, + 'title' => $title, + 'metadata' => $metadata, + ]; + } + + return $rows; + } + + /** + * @param array{url: string, title: ?string, metadata: array}[] $rows + * + * @return array{created: int, updated: int} + */ + public function upsertRows(array $rows): array + { + /** @var ImportUrlRepository $repository */ + $repository = $this->entityManager->getRepository(ImportUrl::class); + $created = 0; + $updated = 0; + $now = CarbonImmutable::now(); + + foreach ($rows as $index => $row) { + $url = $row['url'] ?? ''; + if ($url === '') { + continue; + } + + $importUrl = $repository->findOneByUrl($url); + if ($importUrl) { + if (!empty($row['title'])) { + $importUrl->setTitle($row['title']); + } + if (!empty($row['metadata'])) { + $existing = $importUrl->getMetadata() ?: []; + $importUrl->setMetadata(array_merge($existing, $row['metadata'])); + } + ++$updated; + } else { + $importUrl = new ImportUrl(); + $importUrl->setUrl($url); + $importUrl->setTitle($row['title'] ?? null); + $importUrl->setMetadata($row['metadata'] ?? null); + $importUrl->setDateAdded($now); + ++$created; + } + + $this->entityManager->persist($importUrl); + + if (($index + 1) % self::CHUNK_SIZE === 0) { + $this->entityManager->flush(); + } + } + + $this->entityManager->flush(); + + return ['created' => $created, 'updated' => $updated]; + } + + public static function suggestUrlColumn(array $headers): ?string + { + foreach (['Address', 'URL', 'url', 'Uri', 'URI'] as $name) { + if (in_array($name, $headers, true)) { + return $name; + } + } + + return $headers[0] ?? null; + } + + public static function suggestTitleColumn(array $headers): ?string + { + foreach (['Title 1', 'Title', 'title', 'H1-1'] as $name) { + if (in_array($name, $headers, true)) { + return $name; + } + } + + return null; + } + + protected function createReader(string $filePath): Reader + { + $reader = Reader::createFromPath($filePath, 'r'); + $reader->setHeaderOffset(0); + $reader->skipInputBOM(); + + return $reader; + } + + /** + * @param array $record + * @param array{column: string, operator: string, value: string}[] $filters + */ + protected function matchesFilters(array $record, array $filters): bool + { + foreach ($filters as $filter) { + $column = $filter['column'] ?? ''; + $operator = $filter['operator'] ?? 'equals'; + $value = (string) ($filter['value'] ?? ''); + if ($column === '') { + continue; + } + + $cell = (string) ($record[$column] ?? ''); + if ($operator === 'contains') { + if (mb_stripos($cell, $value) === false) { + return false; + } + } elseif ($cell !== $value) { + return false; + } + } + + return true; + } +} diff --git a/src/Entity/ImportUrl.php b/src/Entity/ImportUrl.php new file mode 100644 index 0000000..aa115c6 --- /dev/null +++ b/src/Entity/ImportUrl.php @@ -0,0 +1,270 @@ +id; + } + + /** + * @return string + */ + public function getUrl(): string + { + return $this->url; + } + + /** + * @param string $url + */ + public function setUrl(string $url): void + { + $this->url = $url; + } + + /** + * @return string|null + */ + public function getTitle(): ?string + { + return $this->title; + } + + /** + * @param string|null $title + */ + public function setTitle(?string $title): void + { + $this->title = $title; + } + + /** + * @return Batch|null + */ + public function getBatch(): ?Batch + { + return $this->batch; + } + + /** + * @param Batch|null $batch + */ + public function setBatch(?Batch $batch): void + { + $this->batch = $batch; + } + + /** + * @return int|null + */ + public function getImportedCID(): ?int + { + return $this->importedCID; + } + + /** + * @param int|null $importedCID + */ + public function setImportedCID(?int $importedCID): void + { + $this->importedCID = $importedCID; + } + + public function getImportedPage(): ?Page + { + if (!$this->importedCID) { + return null; + } + + $page = Page::getByID($this->importedCID); + if ($page && !$page->isError()) { + return $page; + } + + return null; + } + + /** + * @param Page $page + */ + public function setImportedPage(Page $page): void + { + $this->setImportedCID($page->getCollectionID()); + } + + /** + * @return \DateTimeImmutable|null + */ + public function getImportDate(): ?\DateTimeImmutable + { + return $this->importDate; + } + + /** + * @param \DateTimeImmutable|null $importDate + */ + public function setImportDate(?\DateTimeImmutable $importDate): void + { + $this->importDate = $importDate; + } + + /** + * @return \DateTimeImmutable + */ + public function getDateAdded(): \DateTimeImmutable + { + return $this->dateAdded; + } + + /** + * @param \DateTimeImmutable $dateAdded + */ + public function setDateAdded(\DateTimeImmutable $dateAdded): void + { + $this->dateAdded = $dateAdded; + } + + /** + * @return array|null + */ + public function getMetadata(): ?array + { + return $this->metadata; + } + + /** + * @param array|null $metadata + */ + public function setMetadata(?array $metadata): void + { + $this->metadata = $metadata; + } + + public function isManuallyImported(): bool + { + return ($this->metadata['manual_import'] ?? false) === true; + } + + public function setManuallyImported(bool $manuallyImported): void + { + $metadata = $this->metadata ?: []; + if ($manuallyImported) { + $metadata['manual_import'] = true; + } else { + unset($metadata['manual_import']); + } + $this->metadata = $metadata ?: null; + } + + public function getStatus(): string + { + if ($this->getImportedPage()) { + return self::STATUS_IMPORTED; + } + + if ($this->batch) { + return self::STATUS_ASSIGNED; + } + + return self::STATUS_NOT_YET; + } + + public function getStatusLabel(): string + { + switch ($this->getStatus()) { + case self::STATUS_IMPORTED: + return t('Imported'); + case self::STATUS_ASSIGNED: + return t('Assigned to batch'); + default: + return t('Not yet'); + } + } + + /** + * @return array + */ + public static function getStatusOptions(): array + { + return [ + self::STATUS_NOT_YET => t('Not yet'), + self::STATUS_ASSIGNED => t('Assigned to batch'), + self::STATUS_IMPORTED => t('Imported'), + ]; + } +} diff --git a/src/Install/Installer.php b/src/Install/Installer.php index 35273f9..aeea492 100644 --- a/src/Install/Installer.php +++ b/src/Install/Installer.php @@ -33,6 +33,7 @@ private function installSinglePages(): void '/dashboard/system/content_importer/batches' => 'Batches', '/dashboard/system/content_importer/batches/logs' => 'Batch Logs', '/dashboard/system/content_importer/batches/file_logs' => 'File Logs', + '/dashboard/system/content_importer/urls' => 'URLs', '/dashboard/system/content_importer/list_importer' => 'List Importer', '/dashboard/system/content_importer/config' => 'Config', ]; diff --git a/src/Publisher/BatchPublisher.php b/src/Publisher/BatchPublisher.php index fe1a070..65e22cf 100644 --- a/src/Publisher/BatchPublisher.php +++ b/src/Publisher/BatchPublisher.php @@ -22,6 +22,7 @@ use Macareux\ContentImporter\Entity\ImportBatchLog; use Macareux\ContentImporter\Http\Crawler; use Macareux\ContentImporter\Publisher\Block\BlockPublisherManager; +use Macareux\ContentImporter\Service\ImportUrlStatusSync; use Psr\Log\LoggerInterface; class BatchPublisher implements ApplicationAwareInterface @@ -132,6 +133,10 @@ public function publish(string $sourcePath) $log->setImportDate(CarbonImmutable::now()); $this->entityManager->persist($log); $this->entityManager->flush(); + + /** @var ImportUrlStatusSync $statusSync */ + $statusSync = $this->app->make(ImportUrlStatusSync::class); + $statusSync->markImportedFromLog($log); } else { $this->error->add(t('Failed to start importing.')); } diff --git a/src/Repository/ImportUrlRepository.php b/src/Repository/ImportUrlRepository.php new file mode 100644 index 0000000..257fcb1 --- /dev/null +++ b/src/Repository/ImportUrlRepository.php @@ -0,0 +1,41 @@ +findOneBy(['url' => $url]); + } + + /** + * @return ImportUrl[] + */ + public function findByBatch(Batch $batch): array + { + return $this->findBy(['batch' => $batch]); + } + + /** + * @param string[] $urls + * + * @return ImportUrl[] + */ + public function findByUrls(array $urls): array + { + if ($urls === []) { + return []; + } + + return $this->createQueryBuilder('u') + ->where('u.url IN (:urls)') + ->setParameter('urls', $urls) + ->getQuery() + ->getResult(); + } +} diff --git a/src/Search/ImportUrlList.php b/src/Search/ImportUrlList.php new file mode 100644 index 0000000..b32f7bf --- /dev/null +++ b/src/Search/ImportUrlList.php @@ -0,0 +1,93 @@ +make(EntityManagerInterface::class); + } + + public function createQuery() + { + $this->query->select('u')->from(ImportUrl::class, 'u')->leftJoin('u.batch', 'b'); + } + + public function getResult($mixed) + { + return $mixed; + } + + /** + * {@inheritdoc} + */ + public function getTotalResults() + { + $count = 0; + $query = $this->query->select('count(distinct u.id)') + ->setMaxResults(1)->resetDQLParts(['groupBy', 'orderBy']); + + try { + $count = $query->getQuery()->getSingleScalarResult(); + } catch (NoResultException $e) { + } catch (NonUniqueResultException $e) { + } + + return $count; + } + + /** + * {@inheritdoc} + */ + public function getPaginationAdapter() + { + return new QueryAdapter($this->deliverQueryObject()); + } + + public function filterByKeywords(string $keywords): void + { + $keywords = trim($keywords); + if ($keywords === '') { + return; + } + + $this->query->andWhere($this->query->expr()->like('u.url', ':keywords')) + ->setParameter('keywords', '%' . $keywords . '%'); + } + + public function filterByBatchId(int $batchId): void + { + $this->query->andWhere('b.id = :batchId') + ->setParameter('batchId', $batchId); + } + + public function filterByStatus(string $status): void + { + switch ($status) { + case ImportUrl::STATUS_IMPORTED: + $this->query->andWhere('u.importedCID IS NOT NULL'); + break; + case ImportUrl::STATUS_ASSIGNED: + $this->query->andWhere('u.batch IS NOT NULL') + ->andWhere('u.importedCID IS NULL'); + break; + case ImportUrl::STATUS_NOT_YET: + $this->query->andWhere('u.batch IS NULL') + ->andWhere('u.importedCID IS NULL'); + break; + } + } +} diff --git a/src/Service/ImportUrlStatusSync.php b/src/Service/ImportUrlStatusSync.php new file mode 100644 index 0000000..d67bb06 --- /dev/null +++ b/src/Service/ImportUrlStatusSync.php @@ -0,0 +1,91 @@ +entityManager = $entityManager; + } + + public function markImportedFromLog(ImportBatchLog $log): void + { + /** @var ImportUrlRepository $repository */ + $repository = $this->entityManager->getRepository(ImportUrl::class); + $importUrl = $repository->findOneByUrl($log->getOriginal()); + if (!$importUrl) { + return; + } + + $page = $log->getImportedPage(); + if ($page) { + $importUrl->setImportedPage($page); + } else { + $importUrl->setImportedCID($log->getImportedCID()); + } + $importUrl->setImportDate($log->getImportDate()); + $importUrl->setManuallyImported(false); + if ($log->getBatch()) { + $importUrl->setBatch($log->getBatch()); + } + $this->entityManager->persist($importUrl); + $this->entityManager->flush(); + } + + /** + * @return int number of URLs updated + */ + public function refreshFromLogs(): int + { + /** @var ImportUrlRepository $urlRepository */ + $urlRepository = $this->entityManager->getRepository(ImportUrl::class); + /** @var ImportBatchLogRepository $logRepository */ + $logRepository = $this->entityManager->getRepository(ImportBatchLog::class); + + $updated = 0; + /** @var ImportUrl $importUrl */ + foreach ($urlRepository->findAll() as $importUrl) { + if ($importUrl->isManuallyImported()) { + continue; + } + + $log = $logRepository->findOneByOriginal($importUrl->getUrl()); + if (!$log) { + continue; + } + + $page = $log->getImportedPage(); + if ($page) { + $importUrl->setImportedPage($page); + } else { + $importUrl->setImportedCID($log->getImportedCID()); + } + $importUrl->setImportDate($log->getImportDate()); + if ($log->getBatch() && !$importUrl->getBatch()) { + $importUrl->setBatch($log->getBatch()); + } + $this->entityManager->persist($importUrl); + ++$updated; + + if ($updated % 50 === 0) { + $this->entityManager->flush(); + } + } + + $this->entityManager->flush(); + + return $updated; + } +} diff --git a/src/Service/UrlBatchAssigner.php b/src/Service/UrlBatchAssigner.php new file mode 100644 index 0000000..fe88ba7 --- /dev/null +++ b/src/Service/UrlBatchAssigner.php @@ -0,0 +1,161 @@ +entityManager = $entityManager; + } + + /** + * @param ImportUrl[] $urls + * @param string[] $errors + * + * @return int number of URLs assigned + */ + public function assignUrlsToBatch(array $urls, Batch $batch, array &$errors = []): int + { + $documentRoot = $batch->getDocumentRoot(); + $assigned = 0; + + foreach ($urls as $importUrl) { + if (!$importUrl instanceof ImportUrl) { + continue; + } + + $url = $importUrl->getUrl(); + if ($documentRoot !== '' && strpos($url, $documentRoot) !== 0) { + $errors[] = t('URL does not match batch document root: %s', $url); + continue; + } + + $previousBatch = $importUrl->getBatch(); + if ($previousBatch && $previousBatch->getId() !== $batch->getId()) { + $this->removeUrlFromSourcePath($previousBatch, $url); + } + + $this->appendUrlToSourcePath($batch, $url); + $importUrl->setBatch($batch); + $this->entityManager->persist($importUrl); + ++$assigned; + } + + $this->entityManager->persist($batch); + $this->entityManager->flush(); + + return $assigned; + } + + /** + * @param ImportUrl[] $urls + * + * @return int number of URLs unassigned + */ + public function unassignUrls(array $urls): int + { + $unassigned = 0; + + foreach ($urls as $importUrl) { + if (!$importUrl instanceof ImportUrl) { + continue; + } + + $batch = $importUrl->getBatch(); + if (!$batch) { + continue; + } + + $this->removeUrlFromSourcePath($batch, $importUrl->getUrl()); + $importUrl->setBatch(null); + $this->entityManager->persist($importUrl); + $this->entityManager->persist($batch); + ++$unassigned; + } + + $this->entityManager->flush(); + + return $unassigned; + } + + /** + * Sync inventory assignments from a batch's sourcePath text. + */ + public function syncFromBatchSourcePath(Batch $batch): void + { + /** @var ImportUrlRepository $repository */ + $repository = $this->entityManager->getRepository(ImportUrl::class); + + $paths = []; + foreach (preg_split('/\R/', $batch->getSourcePath()) ?: [] as $line) { + $line = trim($line); + if ($line !== '') { + $paths[] = $line; + } + } + $paths = array_unique($paths); + + $currentlyAssigned = $repository->findByBatch($batch); + foreach ($currentlyAssigned as $importUrl) { + if (!in_array($importUrl->getUrl(), $paths, true)) { + $importUrl->setBatch(null); + $this->entityManager->persist($importUrl); + } + } + + if ($paths !== []) { + foreach ($repository->findByUrls($paths) as $importUrl) { + $previousBatch = $importUrl->getBatch(); + if ($previousBatch && $previousBatch->getId() !== $batch->getId()) { + $this->removeUrlFromSourcePath($previousBatch, $importUrl->getUrl()); + $this->entityManager->persist($previousBatch); + } + $importUrl->setBatch($batch); + $this->entityManager->persist($importUrl); + } + } + + $this->entityManager->flush(); + } + + protected function appendUrlToSourcePath(Batch $batch, string $url): void + { + $lines = []; + foreach (preg_split('/\R/', $batch->getSourcePath()) ?: [] as $line) { + $line = trim($line); + if ($line !== '') { + $lines[] = $line; + } + } + + if (!in_array($url, $lines, true)) { + $lines[] = $url; + } + + $batch->setSourcePath(implode(PHP_EOL, $lines)); + } + + protected function removeUrlFromSourcePath(Batch $batch, string $url): void + { + $lines = []; + foreach (preg_split('/\R/', $batch->getSourcePath()) ?: [] as $line) { + $line = trim($line); + if ($line !== '' && $line !== $url) { + $lines[] = $line; + } + } + + $batch->setSourcePath(implode(PHP_EOL, $lines)); + } +} diff --git a/src/Traits/FileImporterTrait.php b/src/Traits/FileImporterTrait.php index 02b9b2b..192b407 100644 --- a/src/Traits/FileImporterTrait.php +++ b/src/Traits/FileImporterTrait.php @@ -137,6 +137,8 @@ public function importFile($file): Version $file = $this->getDocumentRoot() . $file; } + $file = $this->normalizeDotSegments($file); + $app = Application::getFacadeApplication(); /** @var EntityManagerInterface $entityManager */ $entityManager = $app->make(EntityManagerInterface::class); @@ -224,6 +226,74 @@ public function validateFile(string $path, array $extensions = []): bool return true; } + /** + * Resolve `.` and `..` segments in the path part of a URL or file path, + * e.g. https://example.com/hosp/about/../assets/file.pdf + * becomes https://example.com/hosp/assets/file.pdf. + * + * @param string $file URL or file path + * + * @return string + */ + private function normalizeDotSegments(string $file): string + { + if (strpos($file, './') === false) { + return $file; + } + + $parts = parse_url($file); + if ($parts === false || !isset($parts['path'])) { + return $file; + } + + $path = $parts['path']; + $isAbsolute = strpos($path, '/') === 0; + $segments = []; + foreach (explode('/', $path) as $segment) { + if ($segment === '' || $segment === '.') { + continue; + } + if ($segment === '..') { + if (!empty($segments) && end($segments) !== '..') { + array_pop($segments); + } elseif (!$isAbsolute) { + // Keep leading `..` segments of relative paths, since they can't be resolved + $segments[] = '..'; + } + } else { + $segments[] = $segment; + } + } + $normalizedPath = ($isAbsolute ? '/' : '') . implode('/', $segments); + + $result = ''; + if (isset($parts['scheme'])) { + $result .= $parts['scheme'] . '://'; + } + if (isset($parts['host'])) { + if (isset($parts['user'])) { + $result .= $parts['user']; + if (isset($parts['pass'])) { + $result .= ':' . $parts['pass']; + } + $result .= '@'; + } + $result .= $parts['host']; + if (isset($parts['port'])) { + $result .= ':' . $parts['port']; + } + } + $result .= $normalizedPath; + if (isset($parts['query'])) { + $result .= '?' . $parts['query']; + } + if (isset($parts['fragment'])) { + $result .= '#' . $parts['fragment']; + } + + return $result; + } + private function getFolders(): array { $folders = []; diff --git a/src/Traits/NormalizeFileUrlTrait.php b/src/Traits/NormalizeFileUrlTrait.php new file mode 100644 index 0000000..ec481cd --- /dev/null +++ b/src/Traits/NormalizeFileUrlTrait.php @@ -0,0 +1,203 @@ +documentRoot; + } + + public function setDocumentRoot(string $documentRoot): void + { + $this->documentRoot = $documentRoot; + } + + public function supportPreview(): bool + { + return true; + } + + public function validateRequest(Request $request): ErrorList + { + $error = new ErrorList(); + $documentRoot = trim((string) $request->get('documentRoot')); + if ($documentRoot === '') { + $error->add(t('Please input document root.')); + + return $error; + } + + $parts = parse_url($documentRoot); + $scheme = isset($parts['scheme']) ? strtolower($parts['scheme']) : ''; + if ($parts === false || !isset($parts['host']) || !in_array($scheme, ['http', 'https'], true)) { + $error->add(t('Document root must be an HTTP or HTTPS URL.')); + } + + return $error; + } + + /** + * Convert a relative URL to an absolute URL resolved against the document + * root, like a browser would: `../` goes up one level from the document + * root path (clamped at the host root), and root-relative paths (`/foo`) + * resolve against the host root. Absolute HTTP(S) URLs are left absolute + * (with path dot-segments normalized). Non-HTTP schemes are unchanged. + */ + protected function normalizeFileUrl(string $url): string + { + $url = trim(urldecode($url)); + if ($url === '') { + return ''; + } + + // Fragment-only or non-path schemes that should not be rewritten + if (strpos($url, '#') === 0) { + return $url; + } + + $parts = parse_url($url); + if ($parts === false) { + return $url; + } + + if (isset($parts['scheme'])) { + $scheme = strtolower($parts['scheme']); + if ($scheme === 'http' || $scheme === 'https') { + return $this->buildUrlFromParts($this->normalizePathInParts($parts)); + } + + return $url; + } + + $documentRoot = rtrim((string) $this->getDocumentRoot(), '/'); + if ($documentRoot === '') { + return $url; + } + + $rootParts = parse_url($documentRoot); + if ($rootParts === false || !isset($rootParts['scheme'], $rootParts['host'])) { + return $url; + } + + $relativePath = $parts['path'] ?? ''; + // parse_url may leave path empty for some relative inputs; fall back to the raw string before query/fragment + if ($relativePath === '' && !isset($parts['host'])) { + $relativePath = preg_replace('/[?#].*$/', '', $url) ?? $url; + } + + if (strpos($relativePath, '/') === 0) { + // Root-relative path: resolve against the host root + $fullPath = $relativePath; + } else { + $rootPath = isset($rootParts['path']) ? (string) $rootParts['path'] : ''; + $fullPath = rtrim($rootPath, '/') . '/' . $relativePath; + } + + $merged = [ + 'scheme' => $rootParts['scheme'], + 'host' => $rootParts['host'], + 'path' => '/' . implode('/', $this->pathToSegments($fullPath)), + ]; + if (isset($rootParts['port'])) { + $merged['port'] = $rootParts['port']; + } + if (isset($rootParts['user'])) { + $merged['user'] = $rootParts['user']; + } + if (isset($rootParts['pass'])) { + $merged['pass'] = $rootParts['pass']; + } + if (isset($parts['query'])) { + $merged['query'] = $parts['query']; + } + if (isset($parts['fragment'])) { + $merged['fragment'] = $parts['fragment']; + } + + return $this->buildUrlFromParts($merged); + } + + /** + * @param array $parts parse_url parts + * + * @return array + */ + private function normalizePathInParts(array $parts): array + { + if (!isset($parts['path'])) { + return $parts; + } + $segments = $this->pathToSegments($parts['path']); + $parts['path'] = '/' . implode('/', $segments); + + return $parts; + } + + /** + * Split a path into segments, resolving `.` and `..`. + * `..` pops the previous segment; excess `..` at the host root is dropped. + * + * @return string[] + */ + private function pathToSegments(string $path): array + { + $segments = []; + foreach (explode('/', $path) as $segment) { + if ($segment === '' || $segment === '.') { + continue; + } + if ($segment === '..') { + if (!empty($segments)) { + array_pop($segments); + } + continue; + } + $segments[] = $segment; + } + + return $segments; + } + + /** + * @param array $parts parse_url-style parts + */ + private function buildUrlFromParts(array $parts): string + { + $result = ''; + if (isset($parts['scheme'])) { + $result .= $parts['scheme'] . '://'; + } + if (isset($parts['host'])) { + if (isset($parts['user'])) { + $result .= $parts['user']; + if (isset($parts['pass'])) { + $result .= ':' . $parts['pass']; + } + $result .= '@'; + } + $result .= $parts['host']; + if (isset($parts['port'])) { + $result .= ':' . $parts['port']; + } + } + $result .= $parts['path'] ?? ''; + if (isset($parts['query'])) { + $result .= '?' . $parts['query']; + } + if (isset($parts['fragment'])) { + $result .= '#' . $parts['fragment']; + } + + return $result; + } +} diff --git a/src/Transformer/NormalizeFileUrlAttributeTransformer.php b/src/Transformer/NormalizeFileUrlAttributeTransformer.php new file mode 100644 index 0000000..a9c5c49 --- /dev/null +++ b/src/Transformer/NormalizeFileUrlAttributeTransformer.php @@ -0,0 +1,54 @@ +getDocumentRoot() ?: $batchItem->getBatch()->getDocumentRoot(); + $manager = $app->make(ElementManager::class); + $manager->get('content_importer/transformer/normalize_file_url', [ + 'form' => $app->make('helper/form'), + 'documentRoot' => $documentRoot, + ], 'md_content_importer')->render(); + } + + public function updateFromRequest(Request $request): void + { + $this->setDocumentRoot(trim((string) $request->get('documentRoot'))); + } + + public function transform(string $input): string + { + return $this->normalizeFileUrl($input); + } +} diff --git a/src/Transformer/NormalizeFileUrlContentTransformer.php b/src/Transformer/NormalizeFileUrlContentTransformer.php new file mode 100644 index 0000000..22d78a7 --- /dev/null +++ b/src/Transformer/NormalizeFileUrlContentTransformer.php @@ -0,0 +1,80 @@ +getDocumentRoot() ?: $batchItem->getBatch()->getDocumentRoot(); + $manager = $app->make(ElementManager::class); + $manager->get('content_importer/transformer/normalize_file_url', [ + 'form' => $app->make('helper/form'), + 'documentRoot' => $documentRoot, + ], 'md_content_importer')->render(); + } + + public function updateFromRequest(Request $request): void + { + $this->setDocumentRoot(trim((string) $request->get('documentRoot'))); + } + + public function transform(string $input): string + { + if (strpos($input, '{CCM:') !== false || strpos($input, 'filter('img')->each(function (Crawler $node) { + $src = $node->attr('src'); + if ($src === null || $src === '') { + return; + } + $normalized = $this->normalizeFileUrl($src); + if ($normalized !== '') { + $node->getNode(0)->setAttribute('src', $normalized); + } + }); + + $crawler->filter('a')->each(function (Crawler $node) { + $href = $node->attr('href'); + if ($href === null || $href === '') { + return; + } + $normalized = $this->normalizeFileUrl($href); + if ($normalized !== '') { + $node->getNode(0)->setAttribute('href', $normalized); + } + }); + + return LinkAbstractor::translateTo($crawler->filter('body')->html()); + } +} diff --git a/src/Transformer/TransformerServiceProvider.php b/src/Transformer/TransformerServiceProvider.php index 379d44c..102c509 100644 --- a/src/Transformer/TransformerServiceProvider.php +++ b/src/Transformer/TransformerServiceProvider.php @@ -20,6 +20,8 @@ public function register() $manager->registerTransformer(new RegexTransformer()); $manager->registerTransformer(new ImageFileAttributeTransformer()); $manager->registerTransformer(new ImageFileContentTransformer()); + $manager->registerTransformer(new NormalizeFileUrlAttributeTransformer()); + $manager->registerTransformer(new NormalizeFileUrlContentTransformer()); $manager->registerTransformer(new TopicsAttributeTransformer()); $manager->registerTransformer(new SelectAttributeTransformer()); From 78cfa5b914c929822621de37961d9b6b02ba7cda Mon Sep 17 00:00:00 2001 From: hissy Date: Sun, 19 Jul 2026 22:05:19 +0900 Subject: [PATCH 2/3] Add CHANGELOG --- CHANGELOG.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..fec0b78 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,19 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +## [1.2.0] - 2026-07-19 + +### Added + +- Added a URL inventory dashboard with filtering, sorting, pagination, and import-status tracking. +- Added CSV URL imports with column mapping, row filters, duplicate updates, and progressive chunk processing. +- Added tools to assign URLs to batches, unassign or remove them, and manually associate URLs with imported pages. +- Added automatic URL status synchronization from batch import logs. +- Added transformers for converting relative file, image, and link URLs to absolute URLs in attributes and HTML content. + +### Changed + +- Kept URL inventory assignments synchronized with batch source paths. +- Normalized `.` and `..` path segments before importing files. +- Updated the package version to 1.2.0. From b56b3e27b37b3a7b4e7b3e00b28e6934504e2d87 Mon Sep 17 00:00:00 2001 From: hissy Date: Tue, 28 Jul 2026 01:48:17 +0900 Subject: [PATCH 3/3] Add filtering option --- .../system/content_importer/urls.php | 18 ++++++++---- .../system/content_importer/urls.php | 2 +- src/Search/ImportUrlList.php | 29 ++++++++++++++++++- 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/controllers/single_page/dashboard/system/content_importer/urls.php b/controllers/single_page/dashboard/system/content_importer/urls.php index 8d6563c..6918eab 100644 --- a/controllers/single_page/dashboard/system/content_importer/urls.php +++ b/controllers/single_page/dashboard/system/content_importer/urls.php @@ -66,15 +66,23 @@ public function view() $list->filterByStatus($status); } - $batchId = (int) $this->request->query->get('batch_id', 0); - if ($batchId > 0) { - $list->filterByBatchId($batchId); + $batchFilter = (string) $this->request->query->get('batch_id', ''); + if ($batchFilter === 'unassigned') { + $list->filterByUnassigned(); + } elseif ($batchFilter === 'manual') { + $list->filterByManual(); + } elseif ($batchFilter !== '' && ctype_digit($batchFilter) && (int) $batchFilter > 0) { + $list->filterByBatchId((int) $batchFilter); } $factory = new PaginationFactory(Request::getInstance()); $pagination = $factory->createPaginationObject($list, PaginationFactory::PERMISSIONED_PAGINATION_STYLE_PAGER); - $batchOptions = ['' => t('** All Batches')]; + $batchOptions = [ + '' => t('** All Batches'), + 'unassigned' => t('Unassigned'), + 'manual' => t('Manual'), + ]; $assignBatchOptions = ['' => t('** Select Batch')]; foreach ($this->getAll(Batch::class) as $batch) { /** @var Batch $batch */ @@ -86,7 +94,7 @@ public function view() $this->set('pagination', $pagination); $this->set('keywords', $keywords); $this->set('status', $status); - $this->set('batchId', $batchId); + $this->set('batchId', $batchFilter); $this->set('sort', $sort); $this->set('direction', $direction); $this->set('statusOptions', ['' => t('** All Statuses')] + ImportUrl::getStatusOptions()); diff --git a/single_pages/dashboard/system/content_importer/urls.php b/single_pages/dashboard/system/content_importer/urls.php index 4d4dc0a..58dad9c 100644 --- a/single_pages/dashboard/system/content_importer/urls.php +++ b/single_pages/dashboard/system/content_importer/urls.php @@ -10,7 +10,7 @@ /** @var \Concrete\Core\Search\Pagination\Pagination $pagination */ /** @var string $keywords */ /** @var string $status */ -/** @var int $batchId */ +/** @var string $batchId */ /** @var string $sort */ /** @var string $direction */ /** @var array $statusOptions */ diff --git a/src/Search/ImportUrlList.php b/src/Search/ImportUrlList.php index b32f7bf..093391d 100644 --- a/src/Search/ImportUrlList.php +++ b/src/Search/ImportUrlList.php @@ -74,6 +74,18 @@ public function filterByBatchId(int $batchId): void ->setParameter('batchId', $batchId); } + public function filterByUnassigned(): void + { + $this->query->andWhere('u.batch IS NULL') + ->andWhere($this->notManuallyImportedExpression()); + } + + public function filterByManual(): void + { + $this->query->andWhere('u.batch IS NULL') + ->andWhere($this->manuallyImportedExpression()); + } + public function filterByStatus(string $status): void { switch ($status) { @@ -86,8 +98,23 @@ public function filterByStatus(string $status): void break; case ImportUrl::STATUS_NOT_YET: $this->query->andWhere('u.batch IS NULL') - ->andWhere('u.importedCID IS NULL'); + ->andWhere('u.importedCID IS NULL') + ->andWhere($this->notManuallyImportedExpression()); break; } } + + private function manuallyImportedExpression(): string + { + $this->query->setParameter('manualImportFlag', '%"manual_import":true%'); + + return 'u.metadata LIKE :manualImportFlag'; + } + + private function notManuallyImportedExpression(): string + { + $this->query->setParameter('manualImportFlag', '%"manual_import":true%'); + + return '(u.metadata IS NULL OR u.metadata NOT LIKE :manualImportFlag)'; + } }