import 'https' import { PrismaClient } from '@prisma/client' import pkg from '@prisma/client' const { PrismaPromise } = pkg; import { performance } from 'perf_hooks' const start = performance.now() const db = new PrismaClient() // Found on Github : https://github.com/prisma/prisma/discussions/19765#discussioncomment-9986300 function bulkUpdate(tableName, entries){ if (entries.length === 0) return db.$executeRawUnsafe(`SELECT 1;`); const fields = Object.keys(entries[0]).filter((key) => key !== 'id'); const setSql = fields .map((field) => `"${field}" = data."${field}"`) .join(', '); const valuesSql = entries .map((entry) => { const values = fields.map((field) => { const value = entry[field]; if (typeof value === 'string') { // Handle strings and escape single quotes return `'${value.replace(/'/g, "''")}'`; } else if (value instanceof Date) { // Convert Date to ISO 8601 string format return `'${value.toISOString()}'`; } else if (value == null) { // Handle null values or undefined return `NULL`; } // Numbers and booleans are used as-is return value; }); return `('${entry.id}', ${values.join(', ')})`; }) .join(', '); const sql = ` UPDATE "${tableName}" SET ${setSql} FROM (VALUES ${valuesSql}) AS data(id, ${fields .map((field) => `"${field}"`) .join(', ')}) WHERE "${tableName}".id::text = data.id; `; return db.$executeRawUnsafe(sql); } console.log("Fetching latest sets list from Scryfall...") const scryfallSets = await fetch('https://api.scryfall.com/sets'); const sets = await scryfallSets.json(); console.log("Fetching BrawlSet Set Codes...") const bsets = await db.bset.findMany({ relationLoadStrategy: "join", include: { sets: true } }) let set_codes = [] bsets.forEach((bset) => { bset.sets.forEach((set) => { set_codes.push(set.code) }) }) async function fetchApiData(url, cards) { const apiData = await fetch(url) const data = await apiData.json() let res_cards = [...cards, ...data.data] if( data.has_more) { console.log(" fetching next page...") return fetchApiData(data.next_page, res_cards) } else { return res_cards } } let scryfallData = [] // Read the data from the exported fr_cards.json extracted from Scryfall Bulk Data console.log("Reading Bulk Data...") let nbr_sets = set_codes.length for(const [index, code] of set_codes.entries()) { console.log("fetching " + code + "... " + index + "/" + nbr_sets) const setCardsData = await fetchApiData("https://api.scryfall.com/cards/search?q=(game%3Apaper)+set%3A" + code, []) scryfallData = [...scryfallData, ...setCardsData] } const two_faced_layouts = ["transform","modal_dfc","double_faced_token","reversible_card"] const total_cards = scryfallData.length console.log("Creating objects to update database... (" + total_cards + " cards found)" ) try { const setsRes = await db.set.findMany() let preUpdateSetIds = [] setsRes.forEach(element => { preUpdateSetIds.push(element.id) }); for (const set of sets.data) { if(!preUpdateSetIds.includes(set.id)){ await db.set.create({ data: { id: set.id, name_en: set.name, sanitized_name: set.name.replace(/[^a-zA-Z0-9]/gim,"-").toLowerCase(), code: set.code, set_type: set.set_type, released_at: set.released_at, icon_svg_uri: set.icon_svg_uri } }) } } // Select already imported cards in database const cardsRes = await db.carte.findMany() let preUpdateCardsIds = [] cardsRes.forEach(element => { preUpdateCardsIds.push(element.id) }); // Define counter for logging let total_inserted = 0 let total_updated = 0 const cardsToAdd = [] const cardsToUpdate = [] // For each card check if we need to upload it to the database for (const carte of scryfallData) { if(!preUpdateCardsIds.includes(carte.id)){ let type = "" const card_type = (carte.type_line == undefined) ? carte.card_faces[0].type_line.toLowerCase() : carte.type_line.toLowerCase() let can_be_commander = (card_type.includes("legendary") && (card_type.includes("creature") || card_type.includes("planeswalker"))) ? true : false if(card_type.includes("creature")){ type = "creature" } else if (card_type.includes("planeswalker")) { type = "planeswalker" } else if (card_type.includes("artifact")) { type = "artifact" } else if (card_type.includes("instant")) { type = "instant" } else if (card_type.includes("enchantment")) { type = "enchantment" } else if (card_type.includes("sorcery")) { type = "sorcery" } else if (card_type.includes("land")) { type = "land" } let cardObject = { id: carte.id, name: carte.name, released_at: carte.released_at, small_image: carte.card_faces[0].image_uris.small, small_image_back: carte.card_faces[1].image_uris.small, normal_image: carte.card_faces[0].image_uris.normal, normal_image_back: carte.card_faces[1].image_uris.normal, type_line: carte.type_line, color_identity: carte.color_identity, set_id: carte.set_id, rarity: carte.rarity, cardmarket_uri: carte.purchase_uris?.cardmarket, price: carte.prices.eur, type: type, sanitized_name: carte.name.replace(/[^a-zA-Z0-9]/gim,"-").toLowerCase(), set_code: carte.set, layout: carte.layout, is_promo: false, can_be_commander: can_be_commander } if(two_faced_layouts.includes(carte.layout)) { cardObject.small_image = carte.card_faces[0].image_uris.small cardObject.small_image_back = carte.card_faces[1].image_uris.small cardObject.normal_image= carte.card_faces[0].image_uris.normal cardObject.normal_image_back= carte.card_faces[1].image_uris.normal } else { cardObject.small_image = carte.image_uris.small cardObject.normal_image= carte.image_uris.normal } cardsToAdd.push(cardObject) total_inserted = total_inserted + 1 } else { cardsToUpdate.push({ id: carte.id, price: carte.prices.eur }) } } console.log("Inserting cards in database...") await db.carte.createMany({ data: cardsToAdd }) console.log("Updating cards in database...") await bulkUpdate("carte", cardsToUpdate) console.log("Un total de " + total_inserted + " cartes ont été insérées.") console.log("Un total de " + total_updated + " cartes ont été mises à jour.") } catch (err) { console.error(err); } const end = performance.now() console.log(`Time taken to generate stats is ${(end - start)/1000}s.`);