Add Scrape Autopilot components

components/scrape_autopilot/README.md

@@ -0,0 +1,11 @@
+# Overview
+
+Scrape Autopilot provides cost-efficient web scraping for public URLs with Markdown, HTML, and plain text outputs. Use it to extract clean page content from websites and feed the results into Pipedream workflows, AI steps, databases, alerts, and reporting pipelines.
+
+Authenticate with your Scrape Autopilot API key from https://www.scrappilot.com/dashboard.
+
+# Example Use Cases
+
+- **AI-ready content extraction**: Scrape a URL as Markdown, then send the clean content to an AI step for summarization, classification, or entity extraction.
+- **Batch website monitoring**: Scrape a short list of URLs on a schedule and compare the returned text or Markdown against previous runs.
+- **Lead and research workflows**: Extract readable page content from company websites, product pages, or public articles, then store structured results in Airtable, Google Sheets, or a database.

components/scrape_autopilot/actions/get-balance/get-balance.mjs

@@ -0,0 +1,26 @@
+import scrapeAutopilot from "../../scrape_autopilot.app.mjs";
+
+export default {
+  name: "Get Balance",
+  description: "Check your Scrape Autopilot credit balance to keep cost-efficient scraping workflows under control.",
+  key: "scrape-ap-get-balance",
+  version: "0.0.1",
+  annotations: {
+    destructiveHint: false,
+    openWorldHint: true,
+    readOnlyHint: true,
+  },
+  type: "action",
+  props: {
+    scrapeAutopilot,
+  },
+  async run({ $ }) {
+    const data = await this.scrapeAutopilot.request($, {
+      method: "GET",
+      path: "/api/status",
+    });
+
+    $.export("$summary", `Credit balance: ${data.credits}`);
+    return data;
+  },
+};

components/scrape_autopilot/actions/scrape-url/scrape-url.mjs

@@ -0,0 +1,69 @@
+import scrapeAutopilot from "../../scrape_autopilot.app.mjs";
+
+const FORMATS = [
+  {
+    label: "Markdown",
+    value: "md",
+  },
+  {
+    label: "HTML",
+    value: "html",
+  },
+  {
+    label: "Text",
+    value: "text",
+  },
+];
+
+export default {
+  name: "Scrape URL",
+  description: "Cost-efficiently scrape one public URL and return Markdown, HTML, or text.",
+  key: "scrape-ap-url",
+  version: "0.0.1",
+  annotations: {
+    destructiveHint: false,
+    openWorldHint: true,
+    readOnlyHint: true,
+  },
+  type: "action",
+  props: {
+    scrapeAutopilot,
+    url: {
+      type: "string",
+      label: "URL",
+      description: "The fully qualified public URL to scrape.",
+    },
+    format: {
+      type: "string",
+      label: "Output Format",
+      description: "The response format to return.",
+      options: FORMATS,
+      optional: true,
+      default: "md",
+    },
+    js: {
+      type: "boolean",
+      label: "Enable JavaScript Rendering",
+      description: "Use JavaScript rendering for dynamic pages. This consumes more credits.",
+      optional: true,
+      default: false,
+    },
+  },
+  async run({ $ }) {
+    const data = await this.scrapeAutopilot.request($, {
+      method: "POST",
+      path: "/api/scrape",
+      headers: {
+        "Content-Type": "application/json",
+      },
+      data: {
+        url: this.url,
+        format: this.format || "md",
+        js: Boolean(this.js),
+      },
+    });
+
+    $.export("$summary", `Scraped ${this.url}`);
+    return data;
+  },
+};

components/scrape_autopilot/actions/scrape-urls/scrape-urls.mjs

@@ -0,0 +1,85 @@
+import scrapeAutopilot from "../../scrape_autopilot.app.mjs";
+
+const MAX_URLS = 10;
+const FORMATS = [
+  {
+    label: "Markdown",
+    value: "md",
+  },
+  {
+    label: "HTML",
+    value: "html",
+  },
+  {
+    label: "Text",
+    value: "text",
+  },
+];
+
+export default {
+  name: "Scrape URLs",
+  description: "Cost-efficiently scrape up to 10 public URLs and return one result per URL.",
+  key: "scrape-ap-urls",
+  version: "0.0.1",
+  annotations: {
+    destructiveHint: false,
+    openWorldHint: true,
+    readOnlyHint: true,
+  },
+  type: "action",
+  props: {
+    scrapeAutopilot,
+    urls: {
+      type: "string[]",
+      label: "URLs",
+      description: "Public URLs to scrape. Maximum 10.",
+    },
+    format: {
+      type: "string",
+      label: "Output Format",
+      description: "The response format to return.",
+      options: FORMATS,
+      optional: true,
+      default: "md",
+    },
+    js: {
+      type: "boolean",
+      label: "Enable JavaScript Rendering",
+      description: "Use JavaScript rendering for dynamic pages. This consumes more credits.",
+      optional: true,
+      default: false,
+    },
+  },
+  async run({ $ }) {
+    const urls = (this.urls || []).map((url) => url.trim()).filter(Boolean);
+
+    if (!urls.length) {
+      throw new Error("Provide at least one URL.");
+    }
+
+    if (urls.length > MAX_URLS) {
+      throw new Error(`Scrape Autopilot batch scraping is limited to ${MAX_URLS} URLs.`);
+    }
+
+    const data = await this.scrapeAutopilot.request($, {
+      method: "POST",
+      path: "/api/scrape",
+      headers: {
+        "Content-Type": "application/json",
+      },
+      data: {
+        urls,
+        format: this.format || "md",
+        js: Boolean(this.js),
+      },
+    });
+
+    $.export(
+      "$summary",
+      `Scraped ${urls.length} URL${urls.length === 1
+        ? ""
+        : "s"}`,
+    );
+    return data;
+  },
+};

components/scrape_autopilot/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "@pipedream/scrape_autopilot",
+  "version": "0.0.1",
+  "description": "Pipedream Scrape Autopilot Components",
+  "main": "scrape_autopilot.app.mjs",
+  "keywords": [
+    "pipedream",
+    "scrape_autopilot",
+    "web-scraping",
+    "markdown"
+  ],
+  "homepage": "https://pipedream.com/apps/scrape_autopilot",
+  "author": "Pipedream <support@pipedream.com> (https://pipedream.com/)",
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@pipedream/platform": "^3.1.1"
+  }
+}

components/scrape_autopilot/scrape_autopilot.app.mjs

@@ -0,0 +1,34 @@
+import { axios } from "@pipedream/platform";
+
+export default {
+  type: "app",
+  app: "scrape_autopilot",
+  propDefinitions: {
+    scrapeAutopilot: {
+      type: "app",
+      app: "scrape_autopilot",
+      label: "Scrape Autopilot",
+      description: "Connect your Scrape Autopilot account.",
+    },
+  },
+  methods: {
+    _baseUrl() {
+      return "https://www.scrappilot.com";
+    },
+    _authHeaders() {
+      return {
+        Authorization: this.$auth.api_key,
+      };
+    },
+    async request($, opts) {
+      return axios($, {
+        ...opts,
+        url: `${this._baseUrl()}${opts.path}`,
+        headers: {
+          ...this._authHeaders(),
+          ...(opts.headers || {}),
+        },
+      });
+    },
+  },
+};