First commit
This commit is contained in:
103
backend/scripts/compile-cards.ts
Normal file
103
backend/scripts/compile-cards.ts
Normal file
@@ -0,0 +1,103 @@
|
||||
import { mkdir, readFile, writeFile } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import type { Card } from '../src/cards.js'
|
||||
import { compileCard } from '../src/effects/compile.js'
|
||||
import { formatCard } from '../src/effects/format.js'
|
||||
import { compileImplementation, readAllOverrides } from '../src/effects/implementations.js'
|
||||
import { PseudoParseError } from '../src/effects/pseudo.js'
|
||||
import type { Ability, CompiledCard } from '../src/effects/types.js'
|
||||
|
||||
const DATA_DIR = join(process.cwd(), 'data')
|
||||
const CARDS_FILE = join(DATA_DIR, 'cards.json')
|
||||
const AST_FILE = join(DATA_DIR, 'card-effects.json')
|
||||
const PSEUDO_FILE = join(DATA_DIR, 'card-effects.pseudo.txt')
|
||||
|
||||
function formatBytes(bytes: number): string {
|
||||
return `${(bytes / 1024 / 1024).toFixed(1)} MB`
|
||||
}
|
||||
|
||||
let cards: Card[]
|
||||
try {
|
||||
cards = JSON.parse(await readFile(CARDS_FILE, 'utf8')) as Card[]
|
||||
} catch {
|
||||
console.error(`Card data not found at ${CARDS_FILE}. Run "npm run import:cards" first.`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const overrides = await readAllOverrides()
|
||||
const cardsById = new Map(cards.map(card => [card.oracle_id, card]))
|
||||
for (const oracleId of overrides.keys()) {
|
||||
if (!cardsById.has(oracleId)) {
|
||||
console.warn(`Ignoring override for unknown oracle id: ${oracleId}`)
|
||||
}
|
||||
}
|
||||
|
||||
const compiled: CompiledCard[] = []
|
||||
const cardCounts = { supported: 0, partial: 0, unsupported: 0 }
|
||||
const reasons = new Map<string, number>()
|
||||
const samples: string[] = []
|
||||
let abilityCount = 0
|
||||
let manualCount = 0
|
||||
let parsedCount = 0
|
||||
|
||||
function abilitiesOf(card: CompiledCard): Ability[] {
|
||||
return card.faces ? card.faces.flatMap(face => face.abilities) : card.abilities
|
||||
}
|
||||
|
||||
function sourceLines(ability: Ability): number {
|
||||
if (ability.kind === 'keyword' || ability.kind === 'keywords') return 1
|
||||
return ability.text.split('\n').length
|
||||
}
|
||||
|
||||
for (const card of cards) {
|
||||
const source = overrides.get(card.oracle_id)
|
||||
let result: CompiledCard
|
||||
try {
|
||||
result = source === undefined ? compileCard(card) : compileImplementation(card, source)
|
||||
} catch (error) {
|
||||
if (error instanceof PseudoParseError) {
|
||||
console.error(`Invalid manual implementation for ${card.name} (${card.oracle_id}):`)
|
||||
console.error(` ${error.message}`)
|
||||
process.exit(1)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
if (result.manual) manualCount++
|
||||
compiled.push(result)
|
||||
cardCounts[result.status]++
|
||||
|
||||
for (const ability of abilitiesOf(result)) {
|
||||
const lines = sourceLines(ability)
|
||||
abilityCount += lines
|
||||
if (ability.kind === 'unsupported') {
|
||||
reasons.set(ability.reason, (reasons.get(ability.reason) ?? 0) + lines)
|
||||
if (samples.length < 10 && ability.reason === 'unmatched line') samples.push(ability.text)
|
||||
} else {
|
||||
parsedCount += lines
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await mkdir(DATA_DIR, { recursive: true })
|
||||
const astOutput = JSON.stringify(compiled)
|
||||
const pseudoOutput = compiled.map(formatCard).join('\n\n')
|
||||
await writeFile(AST_FILE, astOutput)
|
||||
await writeFile(PSEUDO_FILE, pseudoOutput)
|
||||
|
||||
console.log(`Compiled ${compiled.length} cards (${manualCount} with manual implementations).`)
|
||||
console.log(
|
||||
`Cards: ${cardCounts.supported} supported, ${cardCounts.partial} partial, ${cardCounts.unsupported} unsupported.`,
|
||||
)
|
||||
console.log(
|
||||
`Ability lines: ${parsedCount}/${abilityCount} parsed (${((100 * parsedCount) / abilityCount).toFixed(1)}%).`,
|
||||
)
|
||||
console.log('Unsupported reasons:')
|
||||
for (const [reason, count] of [...reasons.entries()].sort((a, b) => b[1] - a[1])) {
|
||||
console.log(` ${reason}: ${count}`)
|
||||
}
|
||||
if (samples.length) {
|
||||
console.log('Sample unsupported lines:')
|
||||
for (const sample of samples) console.log(` - ${sample}`)
|
||||
}
|
||||
console.log(`Wrote ${formatBytes(Buffer.byteLength(astOutput))} to ${AST_FILE}`)
|
||||
console.log(`Wrote ${formatBytes(Buffer.byteLength(pseudoOutput))} to ${PSEUDO_FILE}`)
|
||||
166
backend/scripts/import-cards.ts
Normal file
166
backend/scripts/import-cards.ts
Normal file
@@ -0,0 +1,166 @@
|
||||
import { mkdir, rename, writeFile } from 'node:fs/promises'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { createInterface } from 'node:readline'
|
||||
import { Readable } from 'node:stream'
|
||||
import { createGunzip } from 'node:zlib'
|
||||
import type { Card, CardFace } from '../src/cards.js'
|
||||
|
||||
const BULK_API = 'https://api.scryfall.com/bulk-data'
|
||||
const OUT_FILE = join(process.cwd(), 'data', 'cards.json')
|
||||
const HEADERS = {
|
||||
'User-Agent': 'commander-import/0.1',
|
||||
Accept: 'application/json',
|
||||
}
|
||||
|
||||
interface BulkData {
|
||||
type: string
|
||||
name: string
|
||||
updated_at: string
|
||||
compressed_size: number
|
||||
jsonl_download_uri?: string
|
||||
}
|
||||
|
||||
interface RawFace {
|
||||
name: string
|
||||
mana_cost?: string
|
||||
type_line?: string
|
||||
oracle_text?: string
|
||||
colors?: string[]
|
||||
power?: string
|
||||
toughness?: string
|
||||
loyalty?: string
|
||||
defense?: string
|
||||
image_uris?: { normal?: string }
|
||||
}
|
||||
|
||||
interface RawCard {
|
||||
oracle_id: string
|
||||
name: string
|
||||
layout?: string
|
||||
mana_cost?: string
|
||||
cmc?: number
|
||||
type_line?: string
|
||||
oracle_text?: string
|
||||
colors?: string[]
|
||||
color_identity?: string[]
|
||||
keywords?: string[]
|
||||
rarity?: string
|
||||
power?: string
|
||||
toughness?: string
|
||||
loyalty?: string
|
||||
defense?: string
|
||||
image_uris?: { normal?: string }
|
||||
card_faces?: RawFace[]
|
||||
legalities?: Record<string, string>
|
||||
}
|
||||
|
||||
function toFace(face: RawFace): CardFace {
|
||||
return {
|
||||
name: face.name,
|
||||
mana_cost: face.mana_cost,
|
||||
type_line: face.type_line,
|
||||
oracle_text: face.oracle_text,
|
||||
colors: face.colors ?? [],
|
||||
power: face.power,
|
||||
toughness: face.toughness,
|
||||
loyalty: face.loyalty,
|
||||
defense: face.defense,
|
||||
image_uri: face.image_uris?.normal,
|
||||
}
|
||||
}
|
||||
|
||||
function toCard(raw: RawCard): Card {
|
||||
return {
|
||||
oracle_id: raw.oracle_id,
|
||||
name: raw.name,
|
||||
layout: raw.layout ?? 'normal',
|
||||
cmc: raw.cmc ?? 0,
|
||||
type_line: raw.type_line ?? '',
|
||||
colors: raw.colors ?? [],
|
||||
color_identity: raw.color_identity ?? [],
|
||||
keywords: raw.keywords ?? [],
|
||||
rarity: raw.rarity,
|
||||
legal_formats: Object.entries(raw.legalities ?? {})
|
||||
.filter(([, status]) => status === 'legal' || status === 'restricted')
|
||||
.map(([format]) => format),
|
||||
mana_cost: raw.mana_cost,
|
||||
oracle_text: raw.oracle_text,
|
||||
power: raw.power,
|
||||
toughness: raw.toughness,
|
||||
loyalty: raw.loyalty,
|
||||
defense: raw.defense,
|
||||
image_uri: raw.image_uris?.normal ?? raw.card_faces?.[0]?.image_uris?.normal,
|
||||
card_faces: raw.card_faces?.map(toFace),
|
||||
}
|
||||
}
|
||||
|
||||
function formatBytes(bytes: number): string {
|
||||
return `${(bytes / 1024 / 1024).toFixed(1)} MB`
|
||||
}
|
||||
|
||||
const metaResponse = await fetch(BULK_API, { headers: HEADERS })
|
||||
if (!metaResponse.ok) {
|
||||
throw new Error(`Scryfall bulk-data request failed: ${metaResponse.status}`)
|
||||
}
|
||||
|
||||
const meta = (await metaResponse.json()) as { data: BulkData[] }
|
||||
const bulk = meta.data.find(entry => entry.type === 'oracle_cards')
|
||||
if (!bulk?.jsonl_download_uri) {
|
||||
throw new Error('Scryfall oracle_cards bulk export not found')
|
||||
}
|
||||
|
||||
console.log(
|
||||
`Downloading ${bulk.name} (${formatBytes(bulk.compressed_size)}, updated ${bulk.updated_at})...`,
|
||||
)
|
||||
|
||||
const response = await fetch(bulk.jsonl_download_uri, { headers: HEADERS })
|
||||
if (!response.ok || !response.body) {
|
||||
throw new Error(`Scryfall card download failed: ${response.status}`)
|
||||
}
|
||||
|
||||
const body = response.body as Parameters<typeof Readable.fromWeb>[0]
|
||||
const lines = createInterface({
|
||||
input: Readable.fromWeb(body).pipe(createGunzip()),
|
||||
crlfDelay: Infinity,
|
||||
})
|
||||
|
||||
const cards: Card[] = []
|
||||
const seen = new Set<string>()
|
||||
let skipped = 0
|
||||
|
||||
for await (const line of lines) {
|
||||
if (!line.trim()) continue
|
||||
|
||||
const raw = JSON.parse(line) as RawCard
|
||||
if (!raw.oracle_id || !raw.name) {
|
||||
throw new Error(`Scryfall card is missing oracle_id or name: ${line.slice(0, 120)}`)
|
||||
}
|
||||
if (seen.has(raw.oracle_id)) {
|
||||
throw new Error(`Duplicate oracle_id in Scryfall export: ${raw.oracle_id}`)
|
||||
}
|
||||
seen.add(raw.oracle_id)
|
||||
|
||||
if (raw.legalities?.commander !== 'legal') {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
|
||||
cards.push(toCard(raw))
|
||||
}
|
||||
|
||||
if (cards.length < 15_000) {
|
||||
throw new Error(`Imported only ${cards.length} cards, expected at least 15000`)
|
||||
}
|
||||
|
||||
cards.sort((a, b) => a.name.localeCompare(b.name) || a.oracle_id.localeCompare(b.oracle_id))
|
||||
|
||||
const output = JSON.stringify(cards)
|
||||
const tmpFile = `${OUT_FILE}.tmp`
|
||||
await mkdir(dirname(OUT_FILE), { recursive: true })
|
||||
await writeFile(tmpFile, output)
|
||||
await rename(tmpFile, OUT_FILE)
|
||||
|
||||
console.log(
|
||||
`Imported ${cards.length} Commander-legal cards (skipped ${skipped} other cards, ${seen.size} total).`,
|
||||
)
|
||||
console.log(`Wrote ${formatBytes(Buffer.byteLength(output))} to ${OUT_FILE}`)
|
||||
Reference in New Issue
Block a user