brawlset/app/tools/updateDatabase.mjs

import 'https'
import { PrismaClient } from '@prisma/client'
import pkg from '@prisma/client'
const { PrismaPromise } = pkg;
import { performance } from 'perf_hooks'

const start = performance.now()
const db = new PrismaClient()

// Found on Github : https://github.com/prisma/prisma/discussions/19765#discussioncomment-9986300
function bulkUpdate(tableName, entries){
  if (entries.length === 0) return db.$executeRawUnsafe(`SELECT 1;`);

  const fields = Object.keys(entries[0]).filter((key) => key !== 'id');
  const setSql = fields
    .map((field) => `"${field}" = data."${field}"`)
    .join(', ');

  const valuesSql = entries
    .map((entry) => {
      const values = fields.map((field) => {
        const value = entry[field];
        if (typeof value === 'string') {
          // Handle strings and escape single quotes
          return `'${value.replace(/'/g, "''")}'`;
        } else if (value instanceof Date) {
          // Convert Date to ISO 8601 string format
          return `'${value.toISOString()}'`;
        } else if (value == null) {
          // Handle null values or undefined
          return `NULL`;
        }
        // Numbers and booleans are used as-is
        return value;
      });

      return `('${entry.id}', ${values.join(', ')})`;
    })
    .join(', ');

  const sql = `
    UPDATE "${tableName}"
    SET ${setSql}
    FROM (VALUES ${valuesSql}) AS data(id, ${fields
    .map((field) => `"${field}"`)
    .join(', ')})
    WHERE "${tableName}".id::text = data.id;
  `;

  return db.$executeRawUnsafe(sql);
}

console.log("Fetching latest sets list from Scryfall...")
const scryfallSets = await fetch('https://api.scryfall.com/sets');
const sets = await scryfallSets.json();

console.log("Fetching BrawlSet Set Codes...")
const bsets = await db.bset.findMany({
    relationLoadStrategy: "join",
    include: {
      sets: true
    }
  })


let set_codes = []
bsets.forEach((bset) => {
  bset.sets.forEach((set) => {
    set_codes.push(set.code)
  })
})

async function fetchApiData(url, cards) {
  const apiData = await fetch(url)
  const data = await apiData.json()
  let res_cards = [...cards, ...data.data]
  if( data.has_more) {
    console.log("   fetching next page...")
    return fetchApiData(data.next_page, res_cards)
  } else {
    return res_cards
  }
}

let scryfallData = []

// Read the data from the exported fr_cards.json extracted from Scryfall Bulk Data
console.log("Reading Bulk Data...")
let nbr_sets = set_codes.length
for(const [index, code] of set_codes.entries()) {
  console.log("fetching " + code + "... " + index + "/" + nbr_sets)
  const setCardsData = await fetchApiData("https://api.scryfall.com/cards/search?q=(game%3Apaper)+set%3A" + code, [])
  scryfallData = [...scryfallData, ...setCardsData]
}

const two_faced_layouts = ["transform","modal_dfc","double_faced_token","reversible_card"]

const total_cards = scryfallData.length
console.log("Creating objects to update database... (" + total_cards + " cards found)" )
try {

  const setsRes = await db.set.findMany()
  let preUpdateSetIds = []
  setsRes.forEach(element => {
    preUpdateSetIds.push(element.id)
  });

  for (const set of sets.data) {
    if(!preUpdateSetIds.includes(set.id)){
      await db.set.create({
        data: {
          id: set.id,
          name_en: set.name,
          sanitized_name: set.name.replace(/[^a-zA-Z0-9]/gim,"-").toLowerCase(),
          code: set.code,
          set_type: set.set_type,
          released_at: set.released_at,
          icon_svg_uri: set.icon_svg_uri
        } 
      })
    }
  }


  // Select already imported cards in database
  const cardsRes = await db.carte.findMany()
  let preUpdateCardsIds = []
  cardsRes.forEach(element => {
    preUpdateCardsIds.push(element.id)
  });

  // Define counter for logging
  let total_inserted = 0 
  let total_updated = 0
  
  const cardsToAdd = []
  const cardsToUpdate = []

  // For each card check if we need to upload it to the database
  for (const carte of scryfallData) {
    if(!preUpdateCardsIds.includes(carte.id)){
      let type = ""
      const card_type = (carte.type_line == undefined) ? carte.card_faces[0].type_line.toLowerCase() : carte.type_line.toLowerCase()
      
      let can_be_commander = (card_type.includes("legendary") && (card_type.includes("creature") || card_type.includes("planeswalker"))) ? true : false

      if(card_type.includes("creature")){
        type = "creature"
      } else if (card_type.includes("planeswalker")) {
        type = "planeswalker"
      } else if (card_type.includes("artifact")) {
        type = "artifact"
      } else if (card_type.includes("instant")) {
        type = "instant"
      } else if (card_type.includes("enchantment")) {
        type = "enchantment"
      } else if (card_type.includes("sorcery")) {
        type = "sorcery"
      } else if (card_type.includes("land")) {
        type = "land"
      }

      let cardObject = {
          id: carte.id,
          name: carte.name,
          released_at: carte.released_at,
          small_image: carte.card_faces[0].image_uris.small,
          small_image_back: carte.card_faces[1].image_uris.small,
          normal_image: carte.card_faces[0].image_uris.normal,
          normal_image_back: carte.card_faces[1].image_uris.normal,
          type_line: carte.type_line,
          color_identity: carte.color_identity,
          set_id: carte.set_id,
          rarity: carte.rarity,
          cardmarket_uri: carte.purchase_uris?.cardmarket,
          price: carte.prices.eur,
          type: type,
          sanitized_name: carte.name.replace(/[^a-zA-Z0-9]/gim,"-").toLowerCase(),
          set_code: carte.set,
          layout: carte.layout,
          is_promo: false,
          can_be_commander: can_be_commander
        }
      if(two_faced_layouts.includes(carte.layout)) {
        cardObject.small_image = carte.card_faces[0].image_uris.small
        cardObject.small_image_back = carte.card_faces[1].image_uris.small
        cardObject.normal_image= carte.card_faces[0].image_uris.normal
        cardObject.normal_image_back= carte.card_faces[1].image_uris.normal

      } else {
        cardObject.small_image = carte.image_uris.small
        cardObject.normal_image= carte.image_uris.normal
      }
      
      cardsToAdd.push(cardObject)
      total_inserted = total_inserted + 1
    } else {
      cardsToUpdate.push({ id: carte.id, price: carte.prices.eur })
    }
  }
  console.log("Inserting cards in database...")
  await db.carte.createMany({ data: cardsToAdd }) 

  console.log("Updating cards in database...")
  await bulkUpdate("carte", cardsToUpdate)

  console.log("Un total de " + total_inserted + " cartes ont été insérées.")
  console.log("Un total de " + total_updated + " cartes ont été mises à jour.")
} catch (err) {
   console.error(err);
}
const end = performance.now()
console.log(`Time taken to generate stats is ${(end - start)/1000}s.`);