Feat: Change update database mecanism to use prisma + fetching each set data

app/app/api/account/decks/create/route.ts

@@ -1,6 +1,7 @@
 import { NextResponse, NextRequest } from 'next/server'
 import { validateToken, decryptToken } from '@/lib/jwt'
 import { db } from "@/lib/db"
+import { logging } from '@/lib/logging'
  
 interface orCardFilterProps {
   sanitized_name:    string
@@ -72,24 +73,28 @@ export async function POST(req: NextRequest) {
     // Sort cards to select non promo types first and fallback to promo cards if not found
     cardsData = cardsData.sort((a,b) => +a.is_promo - +b.is_promo)
     
+    const cardsNotFound = []
+
     let allCardFound = true
     if(cardsData.findIndex(cardData => cardData.sanitized_name == commander_name) == -1){
       if(cardsData.findIndex(cardData => cardData.sanitized_name.includes(commander_name)) == -1){
         console.log("Not Found : " + commander_name)
         allCardFound = false
+        cardsNotFound.push(commander_name)
       }
     }
     cards.forEach((card: cardEntryAPIProps) => {
       if(cardsData.findIndex(cardData => cardData.sanitized_name == card.sanitized_name) == -1){
         if(cardsData.findIndex(cardData => cardData.sanitized_name.includes(card.sanitized_name)) == -1 ){
           allCardFound = false
-        console.log("Not Found : " + card.sanitized_name)
+          console.log("Not Found : Bset = " + selected_bset + " , name = " + card.sanitized_name)
+          cardsNotFound.push(card.sanitized_name)
         }
       }
     })
     
     if(!allCardFound) {
-      return NextResponse.json({"message": "Some cards were not found..."},{
+      return NextResponse.json({"message": "Some cards were not found... " + cardsNotFound.join(", ")},{
         status: 401,
       });
     }
@@ -121,7 +126,6 @@ export async function POST(req: NextRequest) {
 
     cards.forEach(async (card: cardEntryAPIProps) => {
       const cardData_id = cardsData.findIndex(cardData => cardData.sanitized_name == card.sanitized_name) == -1 ? cardsData[cardsData.findIndex(cardData => cardData.sanitized_name.includes(card.sanitized_name))].id : cardsData[cardsData.findIndex(cardData => cardData.sanitized_name == card.sanitized_name)].id
-      console.log(card.sanitized_name)
       await db.cartes_dans_deck.create({
         data: {
           amount: card.amount,
@@ -155,6 +159,8 @@ export async function POST(req: NextRequest) {
       }
     })
 
+    logging("Deck created by " + tokenData.username + " : " + name)
+
     return NextResponse.json({"data": deck_complete, "message": "Deck created !"},{
       status: 200,
     });

app/app/api/auth/signin/route.ts

@@ -3,6 +3,7 @@ import { createToken } from '@/lib/jwt'
 import { createHmac } from "crypto"
 
 import { db } from '@/lib/db'
+import { logging } from '@/lib/logging'
 
 const secret = process.env.PASSWORD_SECRET ? process.env.PASSWORD_SECRET : ""
  
@@ -19,6 +20,7 @@ export async function POST(req: NextRequest) {
     if (user !== undefined){
       if(createHmac('sha256',secret).update(password).digest('hex') == user!.password) {
         const token = createToken({data: {username: user!.username, admin: user!.admin, id: user!.id}, maxAge: 60*60*24*7})
+        logging("User " + user!.username + " has connected.")
         return NextResponse.json({"JWT": token},{
           status: 200,
         });

app/app/bset/[bset]/page_content.tsx

@@ -25,15 +25,14 @@ export default function PageContent({bset}: PageContentProps) {
     fetch('/api/json/bset/'+bset+'.json').then((res) => {
       if(res.status == 200) {
         res.json().then((data) => {
-          const limit = 100
+          setCommanderList(data["commander"])
-          setCommanderList(data["commander"].slice(0,limit))
+          setCreatureList(data["creature"])
-          setCreatureList(data["creature"].slice(0,limit))
+          setPlaneswalkerList(data["planeswalker"])
-          setPlaneswalkerList(data["planeswalker"].slice(0,limit))
+          setSorceryList(data["sorcery"])
-          setSorceryList(data["sorcery"].slice(0,limit))
+          setInstantList(data["instant"])
-          setInstantList(data["instant"].slice(0,limit))
+          setEnchantmentList(data["enchantment"])
-          setEnchantmentList(data["enchantment"].slice(0,limit))
+          setLandList(data["land"])
-          setLandList(data["land"].slice(0,limit))
+          setArtifactList(data["artifact"])
-          setArtifactList(data["artifact"].slice(0,limit))
           console.log(data)
         })
       }

app/lib/logging.ts

@@ -0,0 +1,4 @@
+export function logging(message: string) {
+  const date = new Date()
+  console.log("[" + date.getFullYear() + "/" + (date.getMonth()+1).toString().padStart(2,'0') + "/" + date.getDate().toString().padStart(2,'0') + " - " + date.getHours() + ":" + date.getMinutes() + ":" + date.getSeconds() +"] " + message)
+} 

app/tools/createJson.mjs

@@ -160,19 +160,19 @@ async function createJson() {
   writeFileSync(import.meta.dirname + "/../data/misc/bsets.json",JSON.stringify(bsets_list_export), 'utf8')
 
   for (const index of Object.keys(commanderData)) {
-    let JSONToWrite = commanderData[index].sort((a,b) => b.nbr_decks - a.nbr_decks)
+    let JSONToWrite = commanderData[index].sort((a,b) => b.nbr_decks - a.nbr_decks || b.percent_decks - a.percent_decks)
     writeFileSync(import.meta.dirname + "/../data/commander/" + index + ".json",JSON.stringify(JSONToWrite), 'utf8')
   }
 
   for (const index of Object.keys(bset_cards_data_export)) {
     let JSONToWrite = bset_cards_data_export[index]
     for (const type of Object.keys(JSONToWrite)) {
-      JSONToWrite[type] = JSONToWrite[type].sort((a,b) => b.nbr_decks - a.nbr_decks)
+      JSONToWrite[type] = JSONToWrite[type].sort((a,b) => b.nbr_decks - a.nbr_decks || b.percent_decks - a.percent_decks)
     }
       writeFileSync(import.meta.dirname + "/../data/bset/" + index + ".json",JSON.stringify(JSONToWrite), 'utf8')
   }
   const end = performance.now()
-  console.log(`Time taken to generate stats is ${end - start}ms.`);
+  console.log(`Time taken to generate stats is ${(end - start)/1000}s.`);
 }
 
 createJson()

app/tools/updateDatabase.mjs

@@ -1,146 +1,213 @@
-import 'dotenv/config'
 import 'https'
-import fs from 'fs'
+import { PrismaClient } from '@prisma/client'
-import { Readable } from 'stream'
+import pkg from '@prisma/client'
-import { finished } from 'stream/promises'
+const { PrismaPromise } = pkg;
-import pg from 'pg'
+import { performance } from 'perf_hooks'
-const { Client } = pg
 
-console.log("Fetching latest Scryfall Bulk Data URL...")
+const start = performance.now()
-const bulkDataApi = await fetch('https://api.scryfall.com/bulk-data')
+const db = new PrismaClient()
-const bulkDataApiJson = await bulkDataApi.json()
-const bulkDataDownloadUrl = bulkDataApiJson.data.filter((obj) => obj.type == "default_cards")[0].download_uri
 
-console.log("Downloading latest Scryfall Bulk Data...")
+// Found on Github : https://github.com/prisma/prisma/discussions/19765#discussioncomment-9986300
-const stream = fs.createWriteStream(import.meta.dirname + '/data/scryfall_data.json');
+function bulkUpdate(tableName, entries){
-const { body } = await fetch(bulkDataDownloadUrl);
+  if (entries.length === 0) return db.$executeRawUnsafe(`SELECT 1;`);
-await finished(Readable.fromWeb(body).pipe(stream));
+
+  const fields = Object.keys(entries[0]).filter((key) => key !== 'id');
+  const setSql = fields
+    .map((field) => `"${field}" = data."${field}"`)
+    .join(', ');
+
+  const valuesSql = entries
+    .map((entry) => {
+      const values = fields.map((field) => {
+        const value = entry[field];
+        if (typeof value === 'string') {
+          // Handle strings and escape single quotes
+          return `'${value.replace(/'/g, "''")}'`;
+        } else if (value instanceof Date) {
+          // Convert Date to ISO 8601 string format
+          return `'${value.toISOString()}'`;
+        } else if (value == null) {
+          // Handle null values or undefined
+          return `NULL`;
+        }
+        // Numbers and booleans are used as-is
+        return value;
+      });
+
+      return `('${entry.id}', ${values.join(', ')})`;
+    })
+    .join(', ');
+
+  const sql = `
+    UPDATE "${tableName}"
+    SET ${setSql}
+    FROM (VALUES ${valuesSql}) AS data(id, ${fields
+    .map((field) => `"${field}"`)
+    .join(', ')})
+    WHERE "${tableName}".id::text = data.id;
+  `;
+
+  return db.$executeRawUnsafe(sql);
+}
 
 console.log("Fetching latest sets list from Scryfall...")
 const scryfallSets = await fetch('https://api.scryfall.com/sets');
-console.log('Status Code:', scryfallSets.status);
 const sets = await scryfallSets.json();
 
+console.log("Fetching BrawlSet Set Codes...")
+const bsets = await db.bset.findMany({
+    relationLoadStrategy: "join",
+    include: {
+      sets: true
+    }
+  })
+
+
+let set_codes = []
+bsets.forEach((bset) => {
+  bset.sets.forEach((set) => {
+    set_codes.push(set.code)
+  })
+})
+
+async function fetchApiData(url, cards) {
+  const apiData = await fetch(url)
+  const data = await apiData.json()
+  let res_cards = [...cards, ...data.data]
+  if( data.has_more) {
+    console.log("   fetching next page...")
+    return fetchApiData(data.next_page, res_cards)
+  } else {
+    return res_cards
+  }
+}
+
+let scryfallData = []
+
 // Read the data from the exported fr_cards.json extracted from Scryfall Bulk Data
 console.log("Reading Bulk Data...")
-const fileBytes = fs.readFileSync(import.meta.dirname + '/data/scryfall_data.json')
+let nbr_sets = set_codes.length
-let scryfallData = JSON.parse(fileBytes)
+for(const [index, code] of set_codes.entries()) {
-
+  console.log("fetching " + code + "... " + index + "/" + nbr_sets)
-// Connect to postgres database
+  const setCardsData = await fetchApiData("https://api.scryfall.com/cards/search?q=(game%3Apaper)+set%3A" + code, [])
-const client = new Client({
+  scryfallData = [...scryfallData, ...setCardsData]
-  user: process.env.DATABASE_USER,
+}
-  password: process.env.DATABASE_PASSWORD,
-  host: process.env.DATABASE_HOST,
-  port: process.env.DATABASE_PORT,
-  database: process.env.DATABASE_DB
-})
-await client.connect()
-
 
 const two_faced_layouts = ["transform","modal_dfc","double_faced_token","reversible_card"]
-const prohibited_frame_effects = ["extendedart", "showcase", "upsidedowndf", "waxingandwaningmoondfc", "shatteredglass", "convertdfc", "originpwdfc", "draft", "etched", "enchantment","inverted"]
 
-console.log("Starting updating database...")
+const total_cards = scryfallData.length
+console.log("Creating objects to update database... (" + total_cards + " cards found)" )
 try {
-  const setRes = await client.query('SELECT id FROM set')
+
-  const preUpdateSetRows = setRes.rows
+  const setsRes = await db.set.findMany()
   let preUpdateSetIds = []
-  preUpdateSetRows.forEach(element => {
+  setsRes.forEach(element => {
     preUpdateSetIds.push(element.id)
   });
 
   for (const set of sets.data) {
     if(!preUpdateSetIds.includes(set.id)){
-      await client.query('INSERT INTO set(id, name_en, sanitized_name, code, set_type, released_at, icon_svg_uri) VALUES($1, $2, $3, $4, $5, $6, $7)', [set.id, set.name, set.name.replace(/[^a-zA-Z0-9]/gim,"-").toLowerCase(), set.code, set.set_type, set.released_at, set.icon_svg_uri])
+      await db.set.create({
+        data: {
+          id: set.id,
+          name_en: set.name,
+          sanitized_name: set.name.replace(/[^a-zA-Z0-9]/gim,"-").toLowerCase(),
+          code: set.code,
+          set_type: set.set_type,
+          released_at: set.released_at,
+          icon_svg_uri: set.icon_svg_uri
+        } 
+      })
     }
   }
 
 
   // Select already imported cards in database
-  const cardsRes = await client.query('SELECT id FROM carte')
+  const cardsRes = await db.carte.findMany()
-  const preUpdateCardsRows = cardsRes.rows
   let preUpdateCardsIds = []
-  preUpdateCardsRows.forEach(element => {
+  cardsRes.forEach(element => {
     preUpdateCardsIds.push(element.id)
   });
 
   // Define counter for logging
   let total_inserted = 0 
-  let total_skipped = 0
   let total_updated = 0
   
-  const total_cards = scryfallData.length
+  const cardsToAdd = []
+  const cardsToUpdate = []
 
   // For each card check if we need to upload it to the database
   for (const carte of scryfallData) {
-    const total_processed = total_skipped + total_updated + total_inserted
+    if(!preUpdateCardsIds.includes(carte.id)){
-    if ((total_processed) % 1000 == 0) {
+      let type = ""
-      console.log(total_processed + "/" + total_cards) 
+      const card_type = (carte.type_line == undefined) ? carte.card_faces[0].type_line.toLowerCase() : carte.type_line.toLowerCase()
-    }
       
-    if(carte.legalities.commander != "not_legal" && carte.border_color != "borderless" && !carte.full_art && !carte.textless && carte.cardmarket_id && (carte.frame_effects == undefined || prohibited_frame_effects.every((frame_effect) => !carte.frame_effects.includes(frame_effect)))) {
+      let can_be_commander = (card_type.includes("legendary") && (card_type.includes("creature") || card_type.includes("planeswalker"))) ? true : false
 
-      if(!preUpdateCardsIds.includes(carte.id)){
+      if(card_type.includes("creature")){
-        let type = ""
+        type = "creature"
-        const layout = carte.layout
+      } else if (card_type.includes("planeswalker")) {
-        const card_type = (carte.type_line == undefined) ? carte.card_faces[0].type_line.toLowerCase() : carte.type_line.toLowerCase()
+        type = "planeswalker"
-        
+      } else if (card_type.includes("artifact")) {
-        let promo = (carte.promo_types == undefined) ? false : true
+        type = "artifact"
-        let can_be_commander = (card_type.includes("legendary") && (card_type.includes("creature") || card_type.includes("planeswalker"))) ? true : false
+      } else if (card_type.includes("instant")) {
-         
+        type = "instant"
-
+      } else if (card_type.includes("enchantment")) {
-
+        type = "enchantment"
-        if(card_type.includes("creature")){
+      } else if (card_type.includes("sorcery")) {
-          type = "creature"
+        type = "sorcery"
-        } else if (card_type.includes("planeswalker")) {
+      } else if (card_type.includes("land")) {
-          type = "planeswalker"
+        type = "land"
-        } else if (card_type.includes("artifact")) {
-          type = "artifact"
-        } else if (card_type.includes("instant")) {
-          type = "instant"
-        } else if (card_type.includes("enchantment")) {
-          type = "enchantment"
-        } else if (card_type.includes("sorcery")) {
-          type = "sorcery"
-        } else if (card_type.includes("land")) {
-          type = "land"
-        }
-
-        try {
-          if(two_faced_layouts.includes(layout)) {
-            const addingCardsQuery = await client.query('INSERT INTO carte(id, name, released_at, small_image, small_image_back, normal_image, normal_image_back, type_line, color_identity, set_id, rarity, cardmarket_uri, price, type, sanitized_name, set_code, layout, is_promo, can_be_commander) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19)', [carte.id, carte.name, carte.released_at, carte.card_faces[0].image_uris.small, carte.card_faces[1].image_uris.small, carte.card_faces[0].image_uris.normal, carte.card_faces[1].image_uris.normal, carte.type_line, carte.color_identity, carte.set_id, carte.rarity, carte.purchase_uris?.cardmarket, carte.prices.eur, type, carte.name.replace(/[^a-zA-Z0-9]/gim,"-").toLowerCase(), carte.set, layout, promo, can_be_commander])
-
-          } else {
-            const addingCardsQuery = await client.query('INSERT INTO carte(id, name, released_at, small_image, normal_image, type_line, color_identity, set_id, rarity, cardmarket_uri, price, type, sanitized_name, set_code, layout, is_promo, can_be_commander) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)', [carte.id, carte.name, carte.released_at, carte.image_uris.small, carte.image_uris.normal, carte.type_line, carte.color_identity, carte.set_id, carte.rarity, carte.purchase_uris?.cardmarket, carte.prices.eur, type, carte.name.replace(/[^a-zA-Z0-9]/gim,"-").toLowerCase(), carte.set, layout, promo, can_be_commander])
-
-          }
-          total_inserted = total_inserted + 1
-        } catch (err) {
-          console.log(carte.uri)
-          console.log(carte.layout)
-          console.log(err)
-          total_skipped = total_skipped + 1
-        }
-      } else {
-        const query = 'UPDATE "carte" SET "price" = $1 WHERE "id" = $2'
-        try {
-          const updateQuery = await client.query(query, [carte.prices.eur, carte.id])
-          total_updated = total_updated + 1
-        } catch (err) {
-          total_skipped = total_skipped + 1
-          console.log(err)
-          console.log(query)
-        }
       }
+
+      let cardObject = {
+          id: carte.id,
+          name: carte.name,
+          released_at: carte.released_at,
+          small_image: carte.card_faces[0].image_uris.small,
+          small_image_back: carte.card_faces[1].image_uris.small,
+          normal_image: carte.card_faces[0].image_uris.normal,
+          normal_image_back: carte.card_faces[1].image_uris.normal,
+          type_line: carte.type_line,
+          color_identity: carte.color_identity,
+          set_id: carte.set_id,
+          rarity: carte.rarity,
+          cardmarket_uri: carte.purchase_uris?.cardmarket,
+          price: carte.prices.eur,
+          type: type,
+          sanitized_name: carte.name.replace(/[^a-zA-Z0-9]/gim,"-").toLowerCase(),
+          set_code: carte.set,
+          layout: carte.layout,
+          is_promo: false,
+          can_be_commander: can_be_commander
+        }
+      if(two_faced_layouts.includes(carte.layout)) {
+        cardObject.small_image = carte.card_faces[0].image_uris.small
+        cardObject.small_image_back = carte.card_faces[1].image_uris.small
+        cardObject.normal_image= carte.card_faces[0].image_uris.normal
+        cardObject.normal_image_back= carte.card_faces[1].image_uris.normal
+
+      } else {
+        cardObject.small_image = carte.image_uris.small
+        cardObject.normal_image= carte.image_uris.normal
+      }
+      
+      cardsToAdd.push(cardObject)
+      total_inserted = total_inserted + 1
     } else {
-      total_skipped = total_skipped + 1
+      cardsToUpdate.push({ id: carte.id, price: carte.prices.eur })
     }
   }
+  console.log("Inserting cards in database...")
+  await db.carte.createMany({ data: cardsToAdd }) 
+
+  console.log("Updating cards in database...")
+  await bulkUpdate("carte", cardsToUpdate)
+
   console.log("Un total de " + total_inserted + " cartes ont été insérées.")
-  console.log("Un total de " + total_skipped + " cartes ont été ignorées.")
   console.log("Un total de " + total_updated + " cartes ont été mises à jour.")
 } catch (err) {
    console.error(err);
-} finally {
-   await client.end()
 }
+const end = performance.now()
+console.log(`Time taken to generate stats is ${(end - start)/1000}s.`);