initial commit

.gitignore

@@ -0,0 +1,3 @@
+scrappy.db
+scrappy.db-journal
+deno.lock

README.md

@@ -0,0 +1,16 @@
+This thing will download new YouTube videos from channels using yt-dlp and dump
+them into a directory of your choosing.
+
+Dependencies: Deno, yt-dlp
+
+Create your database:
+
+  deno task create
+
+Add a YouTube channel ID and name:
+
+  deno task new "UCLx053rWZxCiYWsBETgdKrQ" "LGR"
+
+Start downloading:
+
+  deno task get "/tank/Videos/rss/YouTube"

deno.jsonc

@@ -0,0 +1,9 @@
+{
+  "tasks": {
+    "dev": "deno run --watch main.ts",
+    "create": "deno run --allow-read=scrappy.db,scrappy.db-journal --allow-write=scrappy.db,scrappy.db-journal main.ts create",
+    "run": "deno run --allow-net=youtube.com main.ts run",
+    "new": "deno run --allow-read=scrappy.db,scrappy.db-journal --allow-write=scrappy.db,scrappy.db-journal main.ts new",
+    "get": "deno run --allow-net=www.youtube.com --allow-run --allow-read=scrappy.db,scrappy.db-journal --allow-write=scrappy.db,scrappy.db-journal main.ts get"
+  }
+}

main.ts

@@ -0,0 +1,63 @@
+import { DB } from "https://deno.land/x/sqlite/mod.ts";
+import { parse } from "https://deno.land/x/xml/mod.ts"
+import { search } from "https://deno.land/x/jmespath/index.ts";
+
+const YT_FEED_PREFIX = "https://www.youtube.com/feeds/videos.xml?channel_id=";
+const YT_VID_PREFIX = "https://www.youtube.com/watch?v=";
+
+const db = new DB("scrappy.db");
+
+const command = Deno.args[0];
+
+if (command === "create") {
+    db.execute(`create table channel (id INTEGER PRIMARY KEY AUTOINCREMENT, channel_id text not null unique, channel_name text not null, last_seen_at number not null default 0);`);
+}
+else if (command === "new" && Deno.args.length == 3) {
+    const channelId = Deno.args[1];
+    const name = Deno.args[2];
+    console.debug("args:", channelId, name);
+
+    db.query("insert into channel (channel_id, channel_name) values (?, ?)", [channelId, name]);
+    console.log("Saved.");
+
+    db.close();
+}
+else if (command === "get" && Deno.args.length == 2) {
+    const DEST = Deno.args[1];
+
+    const rows = db.query("select * from channel order by id");
+
+    for (const [id, channelId, name, lastSeenAt]: [number, string, string, number] of rows) {
+      await Deno.mkdir(`${DEST}/${name}`, { recursive: true });
+
+      const res = await fetch(YT_FEED_PREFIX + channelId);
+      const raw = await res.text();
+      const doc = parse(raw);
+      const out = search(doc, `feed.entry[].["yt:videoId", published, title]`);
+
+      let newLastSeenAt = lastSeenAt;
+      for (const [videoId, published, title]: [string, string, string] of out.slice(0, 4).reverse()) {
+        const publishedSeconds = Math.floor(new Date(published).getTime() / 1000);
+        if (publishedSeconds > lastSeenAt) {
+          newLastSeenAt = Math.max(publishedSeconds, newLastSeenAt);;
+          const p = Deno.run({ cmd: [
+            "yt-dlp",
+            "-q",
+            "-f22",
+            `${YT_VID_PREFIX}${videoId}`,
+            "-o",
+            `${DEST}/${name}/%(title)s.%(ext)s`
+          ] });
+
+          const status = await p.status();
+          if (status.code) console.error("Exited with status:", status.code);
+        }
+      }
+
+      if (newLastSeenAt != lastSeenAt) db.query("update channel set last_seen_at = ? where id = ?", [newLastSeenAt, id]);
+    }
+}
+else {
+    console.error("Unrecognized command.", Deno.args);
+    Deno.exit(1);
+}