Merge download workflow into run script
This commit is contained in:
390
run.mjs
390
run.mjs
@@ -1,15 +1,14 @@
|
||||
#!/usr/bin/env node
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { spawn } from 'child_process';
|
||||
import process from 'process';
|
||||
import { chromium } from 'playwright';
|
||||
import http from 'http';
|
||||
import https from 'https';
|
||||
|
||||
const VIEWPORT = {
|
||||
width: 390,
|
||||
height: 844,
|
||||
const VIEWPORT_SIZE = { width: 375, height: 667 };
|
||||
const CONTEXT_OPTIONS = {
|
||||
viewport: VIEWPORT_SIZE,
|
||||
deviceScaleFactor: 3,
|
||||
isMobile: true,
|
||||
hasTouch: true,
|
||||
@@ -22,6 +21,9 @@ const MEDIA_EXTENSIONS = [
|
||||
'.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.bmp', '.ico',
|
||||
];
|
||||
|
||||
const MAX_RETRIES = 2;
|
||||
const TIMEOUT = 30000;
|
||||
|
||||
function getCurrentDate() {
|
||||
const now = new Date();
|
||||
const year = now.getFullYear();
|
||||
@@ -91,59 +93,6 @@ function buildCaptureUrl(pageUrl) {
|
||||
return `https://ca.rrx.cn/v/${pageId}?rrxsrc=2&iframe=1&tpl=1`;
|
||||
}
|
||||
|
||||
async function captureNetworkAndScreenshot(context, captureUrl, pageId, today) {
|
||||
const page = await context.newPage();
|
||||
const requests = new Set();
|
||||
|
||||
page.on('request', (request) => {
|
||||
const resourceType = request.resourceType();
|
||||
if (resourceType === 'image' || resourceType === 'media') {
|
||||
requests.add(request.url());
|
||||
}
|
||||
});
|
||||
|
||||
page.on('response', (response) => {
|
||||
const url = response.url();
|
||||
const mimeType = response.headers()['content-type'] || '';
|
||||
if (isMediaUrl(url, mimeType)) {
|
||||
requests.add(url);
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
console.log('Navigating to:', captureUrl);
|
||||
await page.goto(captureUrl, { waitUntil: 'networkidle', timeout: 30000 });
|
||||
await page.waitForTimeout(3000);
|
||||
|
||||
const pageTitle = sanitizeTitle(await page.title());
|
||||
const destRoot = path.join('downloads', `${today}-${pageTitle}-${pageId}`);
|
||||
await fs.mkdir(destRoot, { recursive: true });
|
||||
|
||||
const screenshotPath = path.join(destRoot, 'index.png');
|
||||
console.log('Taking screenshot...');
|
||||
await page.screenshot({ path: screenshotPath, fullPage: false });
|
||||
console.log('Screenshot saved:', screenshotPath);
|
||||
|
||||
return { mediaUrls: [...requests], pageTitle };
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
}
|
||||
|
||||
function runDownloadScript(pageId, title, urlsFile, sourcesFile) {
|
||||
return new Promise((resolve) => {
|
||||
const args = [
|
||||
'download.mjs',
|
||||
'--page-id', pageId,
|
||||
'--title', title,
|
||||
'--urls', urlsFile,
|
||||
'--sources', sourcesFile,
|
||||
];
|
||||
const proc = spawn('node', args, { stdio: 'inherit' });
|
||||
proc.on('close', resolve);
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchJson(url) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const client = url.startsWith('https:') ? https : http;
|
||||
@@ -186,6 +135,322 @@ async function resolveCdpUrl(cdpUrl) {
|
||||
return data.webSocketDebuggerUrl;
|
||||
}
|
||||
|
||||
function guessExtension(contentType) {
|
||||
if (!contentType) {
|
||||
return '.bin';
|
||||
}
|
||||
const mime = contentType.split(';')[0].trim();
|
||||
const map = {
|
||||
'image/jpeg': '.jpg',
|
||||
'image/png': '.png',
|
||||
'image/gif': '.gif',
|
||||
'image/webp': '.webp',
|
||||
'image/svg+xml': '.svg',
|
||||
'audio/mpeg': '.mp3',
|
||||
'audio/wav': '.wav',
|
||||
'audio/ogg': '.ogg',
|
||||
'audio/aac': '.aac',
|
||||
'video/mp4': '.mp4',
|
||||
'video/webm': '.webm',
|
||||
'video/ogg': '.ogv',
|
||||
'application/octet-stream': '.bin',
|
||||
};
|
||||
return map[mime] || '.bin';
|
||||
}
|
||||
|
||||
function sanitizeFilename(url, contentType) {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
let basename = path.basename(parsed.pathname);
|
||||
|
||||
if (basename.includes('@!')) {
|
||||
basename = basename.split('@!')[0];
|
||||
}
|
||||
|
||||
if (!basename) {
|
||||
basename = 'file';
|
||||
}
|
||||
|
||||
const { name, ext } = path.parse(basename);
|
||||
if (!ext) {
|
||||
return name + guessExtension(contentType);
|
||||
}
|
||||
|
||||
return basename;
|
||||
} catch {
|
||||
return 'file.bin';
|
||||
}
|
||||
}
|
||||
|
||||
async function uniqueName(basePath) {
|
||||
try {
|
||||
await fs.access(basePath);
|
||||
const { dir, name, ext } = path.parse(basePath);
|
||||
let idx = 1;
|
||||
while (true) {
|
||||
const candidate = path.join(dir, `${name}_${idx}${ext}`);
|
||||
try {
|
||||
await fs.access(candidate);
|
||||
idx += 1;
|
||||
} catch {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
return basePath;
|
||||
}
|
||||
}
|
||||
|
||||
async function downloadOne(url, destDir, timeout) {
|
||||
return new Promise((resolve) => {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
const client = parsed.protocol === 'https:' ? https : http;
|
||||
|
||||
const req = client.get(url, { timeout }, async (resp) => {
|
||||
if (resp.statusCode !== 200) {
|
||||
resolve([false, `HTTP ${resp.statusCode}`]);
|
||||
resp.resume();
|
||||
return;
|
||||
}
|
||||
|
||||
const contentType = resp.headers['content-type'];
|
||||
const fname = sanitizeFilename(url, contentType);
|
||||
const target = await uniqueName(path.join(destDir, fname));
|
||||
|
||||
const fileStream = await fs.open(target, 'w');
|
||||
const writeStream = fileStream.createWriteStream();
|
||||
|
||||
let failed = false;
|
||||
let errorMsg = null;
|
||||
|
||||
writeStream.on('error', (err) => {
|
||||
failed = true;
|
||||
errorMsg = err.message;
|
||||
writeStream.close();
|
||||
fileStream.close();
|
||||
});
|
||||
|
||||
writeStream.on('finish', () => {
|
||||
writeStream.close();
|
||||
fileStream.close();
|
||||
});
|
||||
|
||||
writeStream.on('close', () => {
|
||||
if (failed) {
|
||||
fs.unlink(target).catch(() => {});
|
||||
resolve([false, errorMsg]);
|
||||
} else {
|
||||
resolve([true, null]);
|
||||
}
|
||||
});
|
||||
|
||||
resp.pipe(writeStream);
|
||||
});
|
||||
|
||||
req.on('error', (err) => {
|
||||
resolve([false, err.message]);
|
||||
});
|
||||
|
||||
req.on('timeout', () => {
|
||||
req.destroy();
|
||||
resolve([false, 'Request timeout']);
|
||||
});
|
||||
} catch (err) {
|
||||
resolve([false, err.message]);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function deleteZeroByteFiles(destDir) {
|
||||
const removed = [];
|
||||
const files = await fs.readdir(destDir);
|
||||
for (const file of files) {
|
||||
const filePath = path.join(destDir, file);
|
||||
const stats = await fs.stat(filePath);
|
||||
if (stats.isFile() && stats.size === 0) {
|
||||
await fs.unlink(filePath);
|
||||
removed.push(file);
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
function planDownloads(urls) {
|
||||
const plan = {};
|
||||
for (const url of urls) {
|
||||
plan[url] = sanitizeFilename(url, null);
|
||||
}
|
||||
return plan;
|
||||
}
|
||||
|
||||
async function verifyMissing(destDir, plan) {
|
||||
const missing = [];
|
||||
const files = await fs.readdir(destDir);
|
||||
const existing = new Set();
|
||||
|
||||
for (const file of files) {
|
||||
const filePath = path.join(destDir, file);
|
||||
const stats = await fs.stat(filePath);
|
||||
if (stats.isFile()) {
|
||||
existing.add(file);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [url, fname] of Object.entries(plan)) {
|
||||
if (!existing.has(fname)) {
|
||||
const baseName = path.parse(fname).name;
|
||||
const hasPrefix = Array.from(existing).some((name) => name.startsWith(baseName));
|
||||
if (!hasPrefix) {
|
||||
missing.push(url);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return missing;
|
||||
}
|
||||
|
||||
function dedupeAndFilterUrls(urls) {
|
||||
const seen = new Set();
|
||||
const filtered = [];
|
||||
const skipped = [];
|
||||
|
||||
for (const url of urls) {
|
||||
const cleanUrl = url.split('?')[0];
|
||||
if (seen.has(cleanUrl)) {
|
||||
skipped.push(url);
|
||||
continue;
|
||||
}
|
||||
seen.add(cleanUrl);
|
||||
filtered.push(url);
|
||||
}
|
||||
|
||||
return { filtered, skipped };
|
||||
}
|
||||
|
||||
async function saveSourceUrls(destRoot, sourcesPath) {
|
||||
const sources = await fs.readFile(sourcesPath, 'utf-8');
|
||||
await fs.writeFile(
|
||||
path.join(destRoot, 'urls.txt'),
|
||||
sources.trim() + '\n',
|
||||
'utf-8',
|
||||
);
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function captureNetworkAndScreenshot(context, captureUrl, pageId, today) {
|
||||
const page = await context.newPage();
|
||||
const requests = new Set();
|
||||
|
||||
page.on('request', (request) => {
|
||||
const resourceType = request.resourceType();
|
||||
if (resourceType === 'image' || resourceType === 'media') {
|
||||
requests.add(request.url());
|
||||
}
|
||||
});
|
||||
|
||||
page.on('response', (response) => {
|
||||
const url = response.url();
|
||||
const mimeType = response.headers()['content-type'] || '';
|
||||
if (isMediaUrl(url, mimeType)) {
|
||||
requests.add(url);
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
console.log('Navigating to:', captureUrl);
|
||||
await page.goto(captureUrl, { waitUntil: 'networkidle', timeout: 30000 });
|
||||
await page.waitForTimeout(3000);
|
||||
|
||||
await page.setViewportSize(VIEWPORT_SIZE);
|
||||
|
||||
const pageTitle = sanitizeTitle(await page.title());
|
||||
const destRoot = path.join('downloads', `${today}-${pageTitle}-${pageId}`);
|
||||
await fs.mkdir(destRoot, { recursive: true });
|
||||
|
||||
const screenshotPath = path.join(destRoot, 'index.png');
|
||||
console.log('Taking screenshot...');
|
||||
await page.screenshot({ path: screenshotPath, fullPage: false });
|
||||
console.log('Screenshot saved:', screenshotPath);
|
||||
|
||||
return { mediaUrls: [...requests], pageTitle, destRoot };
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function downloadAssets(urls, destRoot, timeout, retries) {
|
||||
const mediaDir = path.join(destRoot, 'media');
|
||||
await fs.mkdir(mediaDir, { recursive: true });
|
||||
|
||||
const { filtered, skipped } = dedupeAndFilterUrls(urls);
|
||||
const plan = planDownloads(filtered);
|
||||
|
||||
console.log(`Starting download of ${filtered.length} URLs...`);
|
||||
if (skipped.length > 0) {
|
||||
console.log(`Skipped ${skipped.length} duplicate URLs after cleaning.`);
|
||||
}
|
||||
|
||||
for (const url of filtered) {
|
||||
console.log(`Downloading: ${url}`);
|
||||
const [ok, err] = await downloadOne(url, mediaDir, timeout);
|
||||
if (!ok) {
|
||||
console.error(`WARN: download failed for ${url}: ${err}`);
|
||||
} else {
|
||||
console.log(`Downloaded: ${url}`);
|
||||
}
|
||||
await sleep(50);
|
||||
}
|
||||
|
||||
const removed = await deleteZeroByteFiles(mediaDir);
|
||||
if (removed.length > 0) {
|
||||
console.log(`Removed zero-byte files: ${removed.join(', ')}`);
|
||||
}
|
||||
|
||||
let missing = await verifyMissing(mediaDir, plan);
|
||||
let attempts = 0;
|
||||
|
||||
while (missing.length > 0 && attempts < retries) {
|
||||
attempts += 1;
|
||||
console.log(`Retry round ${attempts}: missing ${missing.length} files`);
|
||||
|
||||
const stillMissing = [];
|
||||
for (const url of missing) {
|
||||
console.log(`Retrying: ${url}`);
|
||||
const [ok, err] = await downloadOne(url, mediaDir, timeout);
|
||||
if (!ok) {
|
||||
console.error(`WARN: retry failed for ${url}: ${err}`);
|
||||
stillMissing.push(url);
|
||||
} else {
|
||||
console.log(`Downloaded on retry: ${url}`);
|
||||
}
|
||||
}
|
||||
|
||||
const removedRetry = await deleteZeroByteFiles(mediaDir);
|
||||
if (removedRetry.length > 0) {
|
||||
console.log(`Removed zero-byte files after retry: ${removedRetry.join(', ')}`);
|
||||
}
|
||||
|
||||
const retryPlan = {};
|
||||
for (const url of stillMissing) {
|
||||
retryPlan[url] = plan[url];
|
||||
}
|
||||
missing = stillMissing.length > 0 ? await verifyMissing(mediaDir, retryPlan) : [];
|
||||
}
|
||||
|
||||
if (missing.length > 0) {
|
||||
console.error('FAILED to fetch these URLs after retries:');
|
||||
for (const url of missing) {
|
||||
console.error(url);
|
||||
}
|
||||
} else {
|
||||
console.log('All planned files downloaded.');
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const { pageId, pageUrl, cdpUrl } = parseArgs();
|
||||
console.log('Extracted page-id:', pageId);
|
||||
@@ -194,7 +459,6 @@ async function main() {
|
||||
const tempDir = 'temp';
|
||||
await fs.mkdir(tempDir, { recursive: true });
|
||||
|
||||
const urlsFile = path.join(tempDir, 'urls.txt');
|
||||
const sourcesFile = path.join(tempDir, 'sources.txt');
|
||||
await fs.writeFile(sourcesFile, pageUrl + '\n', 'utf-8');
|
||||
|
||||
@@ -209,23 +473,25 @@ async function main() {
|
||||
browser = await chromium.launch({ headless: true });
|
||||
}
|
||||
|
||||
const context = await browser.newContext(VIEWPORT);
|
||||
const context = await browser.newContext(CONTEXT_OPTIONS);
|
||||
|
||||
const captureUrl = buildCaptureUrl(pageUrl);
|
||||
console.log('Capture URL:', captureUrl);
|
||||
|
||||
try {
|
||||
const { mediaUrls, pageTitle } = await captureNetworkAndScreenshot(
|
||||
const { mediaUrls, pageTitle, destRoot } = await captureNetworkAndScreenshot(
|
||||
context,
|
||||
captureUrl,
|
||||
pageId,
|
||||
today,
|
||||
);
|
||||
|
||||
await saveSourceUrls(destRoot, sourcesFile);
|
||||
console.log(`Saved source URLs to ${path.join(destRoot, 'urls.txt')}`);
|
||||
|
||||
console.log('Found', mediaUrls.length, 'media URLs');
|
||||
if (mediaUrls.length > 0) {
|
||||
await fs.writeFile(urlsFile, mediaUrls.join('\n') + '\n', 'utf-8');
|
||||
await runDownloadScript(pageId, pageTitle, urlsFile, sourcesFile);
|
||||
await downloadAssets(mediaUrls, destRoot, TIMEOUT, MAX_RETRIES);
|
||||
} else {
|
||||
console.log('No media URLs found, skipping download');
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user