First commit
This commit is contained in:
166
backend/scripts/import-cards.ts
Normal file
166
backend/scripts/import-cards.ts
Normal file
@@ -0,0 +1,166 @@
|
||||
import { mkdir, rename, writeFile } from 'node:fs/promises'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { createInterface } from 'node:readline'
|
||||
import { Readable } from 'node:stream'
|
||||
import { createGunzip } from 'node:zlib'
|
||||
import type { Card, CardFace } from '../src/cards.js'
|
||||
|
||||
const BULK_API = 'https://api.scryfall.com/bulk-data'
|
||||
const OUT_FILE = join(process.cwd(), 'data', 'cards.json')
|
||||
const HEADERS = {
|
||||
'User-Agent': 'commander-import/0.1',
|
||||
Accept: 'application/json',
|
||||
}
|
||||
|
||||
interface BulkData {
|
||||
type: string
|
||||
name: string
|
||||
updated_at: string
|
||||
compressed_size: number
|
||||
jsonl_download_uri?: string
|
||||
}
|
||||
|
||||
interface RawFace {
|
||||
name: string
|
||||
mana_cost?: string
|
||||
type_line?: string
|
||||
oracle_text?: string
|
||||
colors?: string[]
|
||||
power?: string
|
||||
toughness?: string
|
||||
loyalty?: string
|
||||
defense?: string
|
||||
image_uris?: { normal?: string }
|
||||
}
|
||||
|
||||
interface RawCard {
|
||||
oracle_id: string
|
||||
name: string
|
||||
layout?: string
|
||||
mana_cost?: string
|
||||
cmc?: number
|
||||
type_line?: string
|
||||
oracle_text?: string
|
||||
colors?: string[]
|
||||
color_identity?: string[]
|
||||
keywords?: string[]
|
||||
rarity?: string
|
||||
power?: string
|
||||
toughness?: string
|
||||
loyalty?: string
|
||||
defense?: string
|
||||
image_uris?: { normal?: string }
|
||||
card_faces?: RawFace[]
|
||||
legalities?: Record<string, string>
|
||||
}
|
||||
|
||||
function toFace(face: RawFace): CardFace {
|
||||
return {
|
||||
name: face.name,
|
||||
mana_cost: face.mana_cost,
|
||||
type_line: face.type_line,
|
||||
oracle_text: face.oracle_text,
|
||||
colors: face.colors ?? [],
|
||||
power: face.power,
|
||||
toughness: face.toughness,
|
||||
loyalty: face.loyalty,
|
||||
defense: face.defense,
|
||||
image_uri: face.image_uris?.normal,
|
||||
}
|
||||
}
|
||||
|
||||
function toCard(raw: RawCard): Card {
|
||||
return {
|
||||
oracle_id: raw.oracle_id,
|
||||
name: raw.name,
|
||||
layout: raw.layout ?? 'normal',
|
||||
cmc: raw.cmc ?? 0,
|
||||
type_line: raw.type_line ?? '',
|
||||
colors: raw.colors ?? [],
|
||||
color_identity: raw.color_identity ?? [],
|
||||
keywords: raw.keywords ?? [],
|
||||
rarity: raw.rarity,
|
||||
legal_formats: Object.entries(raw.legalities ?? {})
|
||||
.filter(([, status]) => status === 'legal' || status === 'restricted')
|
||||
.map(([format]) => format),
|
||||
mana_cost: raw.mana_cost,
|
||||
oracle_text: raw.oracle_text,
|
||||
power: raw.power,
|
||||
toughness: raw.toughness,
|
||||
loyalty: raw.loyalty,
|
||||
defense: raw.defense,
|
||||
image_uri: raw.image_uris?.normal ?? raw.card_faces?.[0]?.image_uris?.normal,
|
||||
card_faces: raw.card_faces?.map(toFace),
|
||||
}
|
||||
}
|
||||
|
||||
function formatBytes(bytes: number): string {
|
||||
return `${(bytes / 1024 / 1024).toFixed(1)} MB`
|
||||
}
|
||||
|
||||
const metaResponse = await fetch(BULK_API, { headers: HEADERS })
|
||||
if (!metaResponse.ok) {
|
||||
throw new Error(`Scryfall bulk-data request failed: ${metaResponse.status}`)
|
||||
}
|
||||
|
||||
const meta = (await metaResponse.json()) as { data: BulkData[] }
|
||||
const bulk = meta.data.find(entry => entry.type === 'oracle_cards')
|
||||
if (!bulk?.jsonl_download_uri) {
|
||||
throw new Error('Scryfall oracle_cards bulk export not found')
|
||||
}
|
||||
|
||||
console.log(
|
||||
`Downloading ${bulk.name} (${formatBytes(bulk.compressed_size)}, updated ${bulk.updated_at})...`,
|
||||
)
|
||||
|
||||
const response = await fetch(bulk.jsonl_download_uri, { headers: HEADERS })
|
||||
if (!response.ok || !response.body) {
|
||||
throw new Error(`Scryfall card download failed: ${response.status}`)
|
||||
}
|
||||
|
||||
const body = response.body as Parameters<typeof Readable.fromWeb>[0]
|
||||
const lines = createInterface({
|
||||
input: Readable.fromWeb(body).pipe(createGunzip()),
|
||||
crlfDelay: Infinity,
|
||||
})
|
||||
|
||||
const cards: Card[] = []
|
||||
const seen = new Set<string>()
|
||||
let skipped = 0
|
||||
|
||||
for await (const line of lines) {
|
||||
if (!line.trim()) continue
|
||||
|
||||
const raw = JSON.parse(line) as RawCard
|
||||
if (!raw.oracle_id || !raw.name) {
|
||||
throw new Error(`Scryfall card is missing oracle_id or name: ${line.slice(0, 120)}`)
|
||||
}
|
||||
if (seen.has(raw.oracle_id)) {
|
||||
throw new Error(`Duplicate oracle_id in Scryfall export: ${raw.oracle_id}`)
|
||||
}
|
||||
seen.add(raw.oracle_id)
|
||||
|
||||
if (raw.legalities?.commander !== 'legal') {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
|
||||
cards.push(toCard(raw))
|
||||
}
|
||||
|
||||
if (cards.length < 15_000) {
|
||||
throw new Error(`Imported only ${cards.length} cards, expected at least 15000`)
|
||||
}
|
||||
|
||||
cards.sort((a, b) => a.name.localeCompare(b.name) || a.oracle_id.localeCompare(b.oracle_id))
|
||||
|
||||
const output = JSON.stringify(cards)
|
||||
const tmpFile = `${OUT_FILE}.tmp`
|
||||
await mkdir(dirname(OUT_FILE), { recursive: true })
|
||||
await writeFile(tmpFile, output)
|
||||
await rename(tmpFile, OUT_FILE)
|
||||
|
||||
console.log(
|
||||
`Imported ${cards.length} Commander-legal cards (skipped ${skipped} other cards, ${seen.size} total).`,
|
||||
)
|
||||
console.log(`Wrote ${formatBytes(Buffer.byteLength(output))} to ${OUT_FILE}`)
|
||||
Reference in New Issue
Block a user