Fixes for Australia

README.md

@@ -98,7 +98,7 @@ Used to get invoices from amazon
 ```
 USAGE
   $ docudigger scrape amazon -u <value> -p <value> [--json] [--logLevel trace|debug|info|warn|error] [-d] [-l
-    <value>] [-c <value> -r] [--fileDestinationFolder <value>] [--fileFallbackExentension <value>] [-t <value>]
+    <value>] [-c <value> -r] [--fileDestinationFolder <value>] [--fileFallbackExtension <value>] [-t <value>]
     [--yearFilter <value>] [--pageFilter <value>] [--onlyNew]
 
 FLAGS
@@ -109,8 +109,8 @@ FLAGS
   -r, --recurring
   -t, --tld=<value>                  [default: de] Amazon top level domain
   -u, --username=<value>             (required) Username
-  --fileDestinationFolder=<value>    [default: ./data/] Amazon top level domain
-  --fileFallbackExentension=<value>  [default: .pdf] Amazon top level domain
+  --fileDestinationFolder=<value>    [default: ./data/] Where to save the invoices
+  --fileFallbackExtension=<value>    [default: .pdf] File extension for unknown file type
   --logLevel=<option>                [default: info] Specify level for logging.
                                      <options: trace|debug|info|warn|error>
   --onlyNew                          Gets only new invoices

docs/scrape.md

@@ -34,7 +34,7 @@ Scrapes amazon invoices
 ```
 USAGE
   $ document-scraper scrape amazon -u <value> -p <value> [-l <value>] [-l <value>] [--fileDestinationFolder
-    <value>] [--fileFallbackExentension <value>] [-t <value>] [--yearFilter <value>] [--pageFilter <value>] [-d]
+    <value>] [--fileFallbackExtension <value>] [-t <value>] [--yearFilter <value>] [--pageFilter <value>] [-d]
 
 FLAGS
   -d, --debug
@@ -43,8 +43,8 @@ FLAGS
   -p, --password=<value>             (required) Username
   -t, --tld=<value>                  [default: de] Amazon top level domain
   -u, --username=<value>             (required) Username
-  --fileDestinationFolder=<value>    [default: ./data/] Amazon top level domain
-  --fileFallbackExentension=<value>  [default: .pdf] Amazon top level domain
+  --fileDestinationFolder=<value>    [default: ./data/] Where to save the invoices
+  --fileFallbackExtension=<value>    [default: .pdf] File extension for unknown file type
   --pageFilter=<value>               Filters a page
   --yearFilter=<value>               Filters a year
 

src/classes/scrape-command.class.ts

@@ -29,7 +29,7 @@ export abstract class ScrapeCommand<
     `--disable-setuid-sandbox`,
   ];
 
-  public selectorWaitTimeout = 2000;
+  public selectorWaitTimeout = 10000;
 
   protected flags!: ScrapeFlags<T>;
   static baseFlags = {
@@ -46,8 +46,8 @@ export abstract class ScrapeCommand<
       description: `Amazon top level domain`,
       env: `FILE_DESTINATION_FOLDER`,
     }),
-    fileFallbackExentension: Flags.string({
-      aliases: [`fileFallbackExentension`],
+    fileFallbackExtension: Flags.string({
+      aliases: [`fileFallbackExtension`],
       default: `.pdf`,
       description: `Amazon top level domain`,
       env: `FILE_FALLBACK_EXTENSION`,

src/commands/scrape/amazon/helpers/file.helper.ts

@@ -36,10 +36,18 @@ export class FileHandler<T extends typeof Command> {
       path.join(this.flags.fileDestinationFolder, this.flags.subFolderForPages ? this.pluginName : ``, `/`),
       `./`
     );
-    const fileExtention = path.extname(invoiceUrl).split(`?`)[0] ?? this.flags.fileFallbackExentension;
+
+    const urlLower = invoiceUrl.toLowerCase();
+    const isHTML = urlLower.includes('.html') && !urlLower.endsWith('.pdf');
+
+    const fileExtension = isHTML
+      ? '.html'
+      : (path.extname(invoiceUrl).split('?')[0] || this.flags.fileFallbackExtension);
+
     const fileName = `${order.date}_AMZ_${order.number}_${invoiceIndex + 1}`;
-    const fullFilePath = path.resolve(destPluginFileFolder, `${fileName}${fileExtention}`);
+    const fullFilePath = path.resolve(destPluginFileFolder, `${fileName}${fileExtension}`);
     const pathNormalized = path.normalize(fullFilePath);
+
     return { destPluginFileFolder, pathNormalized };
   }
 

src/commands/scrape/amazon/index.ts

@@ -217,12 +217,12 @@ export default class Amazon extends ScrapeCommand<typeof Amazon> {
     for (const [invoiceIndex, invoice] of order.invoices.entries()) {
       const invoiceUrl = invoice.url;
       await new Promise(resolve => setTimeout(resolve, 100));
-      const pdfPage = await this.newPage();
-      await pdfPage.goto(invoiceUrl);
+      const page = await this.newPage();
+      await page.goto(invoiceUrl);
       invoice.status = InvoiceStatus.opened;
 
       try {
-        const fileReaderString = await this.fileHandler.getFileReaderString(pdfPage, invoiceUrl);
+        const fileReaderString = await this.fileHandler.getFileReaderString(page, invoiceUrl);
         const fileBuffer = this.fileHandler.getFileBuffer(fileReaderString, invoice, order, invoiceUrl);
         if (fileBuffer) {
           this.logger.debug(`Buffer exists`);
@@ -235,7 +235,7 @@ export default class Amazon extends ScrapeCommand<typeof Amazon> {
       }
 
       this.logger.debug(`Closing invoice page`);
-      await pdfPage.close();
+      await page.close();
     }
   }
 
@@ -273,6 +273,22 @@ export default class Amazon extends ScrapeCommand<typeof Amazon> {
       this.logger.debug(`Got popover ${(orderIndex + 1)} -> ${popover}`);
       const invoiceList = await popover.waitForSelector(this.selectors.invoiceList, { timeout: this.selectorWaitTimeout });
       invoiceUrls = await invoiceList.$$eval(this.selectors.invoiceLinks, (handles: HTMLAnchorElement[]) => handles.map(a => a.href));
+
+      // if no invoices, fall back to the printable order summary link
+      if (invoiceUrls.length === 0) {
+        const printSummaryUrls = await invoiceList.$$eval(
+          'a[href*=".html"]:not([href*="contact.html"])',
+          (handles: HTMLAnchorElement[]) => handles.map(a => a.href)
+        );
+
+        if (printSummaryUrls.length > 0) {
+          this.logger.info(
+            `No invoices found, using Printable Order Summary URL [#${orderIndex + 1}]: ${JSON.stringify(printSummaryUrls)}`
+          );
+          invoiceUrls = printSummaryUrls;
+        }
+      }
+
       this.logger.debug(`Got invoiceUrls ${(orderIndex + 1)} -> ${invoiceUrls}`);
     } catch (ex) {
       this.logger.error(`Couldn't get popover ${popoverSelectorResolved} within ${this.selectorWaitTimeout}ms. Skipping. ${ex.message}`);
@@ -319,13 +335,31 @@ export default class Amazon extends ScrapeCommand<typeof Amazon> {
     exit(this.logger, options.recurring);
   }
 
+  private getLoginPage(tld: string) {
+    switch (tld) {
+      case 'au':
+        return `https://www.amazon.com.au/ap/signin?openid.pape.max_auth_age=0&openid.return_to=https%3A%2F%2Fwww.amazon.com.au%2Fgp%2Fyourstore%2Fhome%3Fpath%3D%252Fgp%252Fyourstore%252Fhome%26signIn%3D1%26useRedirectOnSuccess%3D1%26action%3Dsign-out%26ref_%3Dnav_AccountFlyout_signout&openid.assoc_handle=auflex&openid.mode=checkid_setup&openid.ns=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0`;
+      default:
+        return `https://www.amazon.${tld}/ap/signin?openid.pape.max_auth_age=0&openid.return_to=https%3A%2F%2Fwww.amazon.de%2F%3Fref_%3Dnav_signin&openid.identity=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&openid.assoc_handle=deflex&openid.mode=checkid_setup&openid.claimed_id=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&openid.ns=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0&`;
+    }
+  }
+
+  private getOrderPage(tld: string) {
+    switch (tld) {
+      case 'au':
+        return `https://www.amazon.com.au/gp/css/order-history`;
+      default:
+        return `https://www.amazon.${tld}/gp/css/order-history`;
+    }
+  }
+
   private getSelectors(tld: string): { amazonSelectors: AmazonSelectors, amazon: AmazonDefinition } {
     this.logger.debug(`Getting selectors...`);
     const amazon: AmazonDefinition = {
       lang: null,
       tld,
-      loginPage: `https://www.amazon.${tld}/ap/signin?openid.pape.max_auth_age=0&openid.return_to=https%3A%2F%2Fwww.amazon.de%2F%3Fref_%3Dnav_signin&openid.identity=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&openid.assoc_handle=deflex&openid.mode=checkid_setup&openid.claimed_id=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&openid.ns=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0&`,
-      orderPage: `https://www.amazon.${tld}/gp/css/order-history`
+      loginPage: this.getLoginPage(tld),
+      orderPage: this.getOrderPage(tld)
     };
     return { amazonSelectors, amazon };
   }

src/interfaces/amazon-options.interface.ts

@@ -9,7 +9,7 @@ export interface AmazonOptions {
     username: string,
     password: string,
     fileDestinationFolder: string,
-    fileFallbackExentension: string,
+    fileFallbackExtension: string,
     tld: string,
     yearFilter: number,
     pageFilter: number,