diff --git a/app/app/api/account/decks/create/route.ts b/app/app/api/account/decks/create/route.ts index cea9f61..d28faad 100644 --- a/app/app/api/account/decks/create/route.ts +++ b/app/app/api/account/decks/create/route.ts @@ -1,6 +1,7 @@ import { NextResponse, NextRequest } from 'next/server' import { validateToken, decryptToken } from '@/lib/jwt' import { db } from "@/lib/db" +import { logging } from '@/lib/logging' interface orCardFilterProps { sanitized_name: string @@ -72,24 +73,28 @@ export async function POST(req: NextRequest) { // Sort cards to select non promo types first and fallback to promo cards if not found cardsData = cardsData.sort((a,b) => +a.is_promo - +b.is_promo) + const cardsNotFound = [] + let allCardFound = true if(cardsData.findIndex(cardData => cardData.sanitized_name == commander_name) == -1){ if(cardsData.findIndex(cardData => cardData.sanitized_name.includes(commander_name)) == -1){ console.log("Not Found : " + commander_name) allCardFound = false + cardsNotFound.push(commander_name) } } cards.forEach((card: cardEntryAPIProps) => { if(cardsData.findIndex(cardData => cardData.sanitized_name == card.sanitized_name) == -1){ if(cardsData.findIndex(cardData => cardData.sanitized_name.includes(card.sanitized_name)) == -1 ){ allCardFound = false - console.log("Not Found : " + card.sanitized_name) + console.log("Not Found : Bset = " + selected_bset + " , name = " + card.sanitized_name) + cardsNotFound.push(card.sanitized_name) } } }) if(!allCardFound) { - return NextResponse.json({"message": "Some cards were not found..."},{ + return NextResponse.json({"message": "Some cards were not found... " + cardsNotFound.join(", ")},{ status: 401, }); } @@ -121,7 +126,6 @@ export async function POST(req: NextRequest) { cards.forEach(async (card: cardEntryAPIProps) => { const cardData_id = cardsData.findIndex(cardData => cardData.sanitized_name == card.sanitized_name) == -1 ? cardsData[cardsData.findIndex(cardData => cardData.sanitized_name.includes(card.sanitized_name))].id : cardsData[cardsData.findIndex(cardData => cardData.sanitized_name == card.sanitized_name)].id - console.log(card.sanitized_name) await db.cartes_dans_deck.create({ data: { amount: card.amount, @@ -155,6 +159,8 @@ export async function POST(req: NextRequest) { } }) + logging("Deck created by " + tokenData.username + " : " + name) + return NextResponse.json({"data": deck_complete, "message": "Deck created !"},{ status: 200, }); diff --git a/app/app/api/auth/signin/route.ts b/app/app/api/auth/signin/route.ts index a6a4bd6..442cf2b 100644 --- a/app/app/api/auth/signin/route.ts +++ b/app/app/api/auth/signin/route.ts @@ -3,6 +3,7 @@ import { createToken } from '@/lib/jwt' import { createHmac } from "crypto" import { db } from '@/lib/db' +import { logging } from '@/lib/logging' const secret = process.env.PASSWORD_SECRET ? process.env.PASSWORD_SECRET : "" @@ -19,6 +20,7 @@ export async function POST(req: NextRequest) { if (user !== undefined){ if(createHmac('sha256',secret).update(password).digest('hex') == user!.password) { const token = createToken({data: {username: user!.username, admin: user!.admin, id: user!.id}, maxAge: 60*60*24*7}) + logging("User " + user!.username + " has connected.") return NextResponse.json({"JWT": token},{ status: 200, }); diff --git a/app/app/bset/[bset]/page_content.tsx b/app/app/bset/[bset]/page_content.tsx index cc0bbef..893e554 100644 --- a/app/app/bset/[bset]/page_content.tsx +++ b/app/app/bset/[bset]/page_content.tsx @@ -25,15 +25,14 @@ export default function PageContent({bset}: PageContentProps) { fetch('/api/json/bset/'+bset+'.json').then((res) => { if(res.status == 200) { res.json().then((data) => { - const limit = 100 - setCommanderList(data["commander"].slice(0,limit)) - setCreatureList(data["creature"].slice(0,limit)) - setPlaneswalkerList(data["planeswalker"].slice(0,limit)) - setSorceryList(data["sorcery"].slice(0,limit)) - setInstantList(data["instant"].slice(0,limit)) - setEnchantmentList(data["enchantment"].slice(0,limit)) - setLandList(data["land"].slice(0,limit)) - setArtifactList(data["artifact"].slice(0,limit)) + setCommanderList(data["commander"]) + setCreatureList(data["creature"]) + setPlaneswalkerList(data["planeswalker"]) + setSorceryList(data["sorcery"]) + setInstantList(data["instant"]) + setEnchantmentList(data["enchantment"]) + setLandList(data["land"]) + setArtifactList(data["artifact"]) console.log(data) }) } diff --git a/app/lib/logging.ts b/app/lib/logging.ts new file mode 100644 index 0000000..0b85b71 --- /dev/null +++ b/app/lib/logging.ts @@ -0,0 +1,4 @@ +export function logging(message: string) { + const date = new Date() + console.log("[" + date.getFullYear() + "/" + (date.getMonth()+1).toString().padStart(2,'0') + "/" + date.getDate().toString().padStart(2,'0') + " - " + date.getHours() + ":" + date.getMinutes() + ":" + date.getSeconds() +"] " + message) +} diff --git a/app/tools/createJson.mjs b/app/tools/createJson.mjs index 6a8e457..0181d00 100644 --- a/app/tools/createJson.mjs +++ b/app/tools/createJson.mjs @@ -160,19 +160,19 @@ async function createJson() { writeFileSync(import.meta.dirname + "/../data/misc/bsets.json",JSON.stringify(bsets_list_export), 'utf8') for (const index of Object.keys(commanderData)) { - let JSONToWrite = commanderData[index].sort((a,b) => b.nbr_decks - a.nbr_decks) + let JSONToWrite = commanderData[index].sort((a,b) => b.nbr_decks - a.nbr_decks || b.percent_decks - a.percent_decks) writeFileSync(import.meta.dirname + "/../data/commander/" + index + ".json",JSON.stringify(JSONToWrite), 'utf8') } for (const index of Object.keys(bset_cards_data_export)) { let JSONToWrite = bset_cards_data_export[index] for (const type of Object.keys(JSONToWrite)) { - JSONToWrite[type] = JSONToWrite[type].sort((a,b) => b.nbr_decks - a.nbr_decks) + JSONToWrite[type] = JSONToWrite[type].sort((a,b) => b.nbr_decks - a.nbr_decks || b.percent_decks - a.percent_decks) } writeFileSync(import.meta.dirname + "/../data/bset/" + index + ".json",JSON.stringify(JSONToWrite), 'utf8') } const end = performance.now() - console.log(`Time taken to generate stats is ${end - start}ms.`); + console.log(`Time taken to generate stats is ${(end - start)/1000}s.`); } createJson() diff --git a/app/tools/updateDatabase.mjs b/app/tools/updateDatabase.mjs index 8ff9e1a..b4ad18c 100644 --- a/app/tools/updateDatabase.mjs +++ b/app/tools/updateDatabase.mjs @@ -1,146 +1,213 @@ -import 'dotenv/config' import 'https' -import fs from 'fs' -import { Readable } from 'stream' -import { finished } from 'stream/promises' -import pg from 'pg' -const { Client } = pg +import { PrismaClient } from '@prisma/client' +import pkg from '@prisma/client' +const { PrismaPromise } = pkg; +import { performance } from 'perf_hooks' -console.log("Fetching latest Scryfall Bulk Data URL...") -const bulkDataApi = await fetch('https://api.scryfall.com/bulk-data') -const bulkDataApiJson = await bulkDataApi.json() -const bulkDataDownloadUrl = bulkDataApiJson.data.filter((obj) => obj.type == "default_cards")[0].download_uri +const start = performance.now() +const db = new PrismaClient() -console.log("Downloading latest Scryfall Bulk Data...") -const stream = fs.createWriteStream(import.meta.dirname + '/data/scryfall_data.json'); -const { body } = await fetch(bulkDataDownloadUrl); -await finished(Readable.fromWeb(body).pipe(stream)); +// Found on Github : https://github.com/prisma/prisma/discussions/19765#discussioncomment-9986300 +function bulkUpdate(tableName, entries){ + if (entries.length === 0) return db.$executeRawUnsafe(`SELECT 1;`); + + const fields = Object.keys(entries[0]).filter((key) => key !== 'id'); + const setSql = fields + .map((field) => `"${field}" = data."${field}"`) + .join(', '); + + const valuesSql = entries + .map((entry) => { + const values = fields.map((field) => { + const value = entry[field]; + if (typeof value === 'string') { + // Handle strings and escape single quotes + return `'${value.replace(/'/g, "''")}'`; + } else if (value instanceof Date) { + // Convert Date to ISO 8601 string format + return `'${value.toISOString()}'`; + } else if (value == null) { + // Handle null values or undefined + return `NULL`; + } + // Numbers and booleans are used as-is + return value; + }); + + return `('${entry.id}', ${values.join(', ')})`; + }) + .join(', '); + + const sql = ` + UPDATE "${tableName}" + SET ${setSql} + FROM (VALUES ${valuesSql}) AS data(id, ${fields + .map((field) => `"${field}"`) + .join(', ')}) + WHERE "${tableName}".id::text = data.id; + `; + + return db.$executeRawUnsafe(sql); +} console.log("Fetching latest sets list from Scryfall...") const scryfallSets = await fetch('https://api.scryfall.com/sets'); -console.log('Status Code:', scryfallSets.status); const sets = await scryfallSets.json(); +console.log("Fetching BrawlSet Set Codes...") +const bsets = await db.bset.findMany({ + relationLoadStrategy: "join", + include: { + sets: true + } + }) + + +let set_codes = [] +bsets.forEach((bset) => { + bset.sets.forEach((set) => { + set_codes.push(set.code) + }) +}) + +async function fetchApiData(url, cards) { + const apiData = await fetch(url) + const data = await apiData.json() + let res_cards = [...cards, ...data.data] + if( data.has_more) { + console.log(" fetching next page...") + return fetchApiData(data.next_page, res_cards) + } else { + return res_cards + } +} + +let scryfallData = [] + // Read the data from the exported fr_cards.json extracted from Scryfall Bulk Data console.log("Reading Bulk Data...") -const fileBytes = fs.readFileSync(import.meta.dirname + '/data/scryfall_data.json') -let scryfallData = JSON.parse(fileBytes) - -// Connect to postgres database -const client = new Client({ - user: process.env.DATABASE_USER, - password: process.env.DATABASE_PASSWORD, - host: process.env.DATABASE_HOST, - port: process.env.DATABASE_PORT, - database: process.env.DATABASE_DB -}) -await client.connect() - +let nbr_sets = set_codes.length +for(const [index, code] of set_codes.entries()) { + console.log("fetching " + code + "... " + index + "/" + nbr_sets) + const setCardsData = await fetchApiData("https://api.scryfall.com/cards/search?q=(game%3Apaper)+set%3A" + code, []) + scryfallData = [...scryfallData, ...setCardsData] +} const two_faced_layouts = ["transform","modal_dfc","double_faced_token","reversible_card"] -const prohibited_frame_effects = ["extendedart", "showcase", "upsidedowndf", "waxingandwaningmoondfc", "shatteredglass", "convertdfc", "originpwdfc", "draft", "etched", "enchantment","inverted"] -console.log("Starting updating database...") +const total_cards = scryfallData.length +console.log("Creating objects to update database... (" + total_cards + " cards found)" ) try { - const setRes = await client.query('SELECT id FROM set') - const preUpdateSetRows = setRes.rows + + const setsRes = await db.set.findMany() let preUpdateSetIds = [] - preUpdateSetRows.forEach(element => { + setsRes.forEach(element => { preUpdateSetIds.push(element.id) }); for (const set of sets.data) { if(!preUpdateSetIds.includes(set.id)){ - await client.query('INSERT INTO set(id, name_en, sanitized_name, code, set_type, released_at, icon_svg_uri) VALUES($1, $2, $3, $4, $5, $6, $7)', [set.id, set.name, set.name.replace(/[^a-zA-Z0-9]/gim,"-").toLowerCase(), set.code, set.set_type, set.released_at, set.icon_svg_uri]) + await db.set.create({ + data: { + id: set.id, + name_en: set.name, + sanitized_name: set.name.replace(/[^a-zA-Z0-9]/gim,"-").toLowerCase(), + code: set.code, + set_type: set.set_type, + released_at: set.released_at, + icon_svg_uri: set.icon_svg_uri + } + }) } } // Select already imported cards in database - const cardsRes = await client.query('SELECT id FROM carte') - const preUpdateCardsRows = cardsRes.rows + const cardsRes = await db.carte.findMany() let preUpdateCardsIds = [] - preUpdateCardsRows.forEach(element => { + cardsRes.forEach(element => { preUpdateCardsIds.push(element.id) }); // Define counter for logging let total_inserted = 0 - let total_skipped = 0 let total_updated = 0 - - const total_cards = scryfallData.length + + const cardsToAdd = [] + const cardsToUpdate = [] // For each card check if we need to upload it to the database for (const carte of scryfallData) { - const total_processed = total_skipped + total_updated + total_inserted - if ((total_processed) % 1000 == 0) { - console.log(total_processed + "/" + total_cards) - } + if(!preUpdateCardsIds.includes(carte.id)){ + let type = "" + const card_type = (carte.type_line == undefined) ? carte.card_faces[0].type_line.toLowerCase() : carte.type_line.toLowerCase() + + let can_be_commander = (card_type.includes("legendary") && (card_type.includes("creature") || card_type.includes("planeswalker"))) ? true : false - if(carte.legalities.commander != "not_legal" && carte.border_color != "borderless" && !carte.full_art && !carte.textless && carte.cardmarket_id && (carte.frame_effects == undefined || prohibited_frame_effects.every((frame_effect) => !carte.frame_effects.includes(frame_effect)))) { - - if(!preUpdateCardsIds.includes(carte.id)){ - let type = "" - const layout = carte.layout - const card_type = (carte.type_line == undefined) ? carte.card_faces[0].type_line.toLowerCase() : carte.type_line.toLowerCase() - - let promo = (carte.promo_types == undefined) ? false : true - let can_be_commander = (card_type.includes("legendary") && (card_type.includes("creature") || card_type.includes("planeswalker"))) ? true : false - - - - if(card_type.includes("creature")){ - type = "creature" - } else if (card_type.includes("planeswalker")) { - type = "planeswalker" - } else if (card_type.includes("artifact")) { - type = "artifact" - } else if (card_type.includes("instant")) { - type = "instant" - } else if (card_type.includes("enchantment")) { - type = "enchantment" - } else if (card_type.includes("sorcery")) { - type = "sorcery" - } else if (card_type.includes("land")) { - type = "land" - } - - try { - if(two_faced_layouts.includes(layout)) { - const addingCardsQuery = await client.query('INSERT INTO carte(id, name, released_at, small_image, small_image_back, normal_image, normal_image_back, type_line, color_identity, set_id, rarity, cardmarket_uri, price, type, sanitized_name, set_code, layout, is_promo, can_be_commander) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19)', [carte.id, carte.name, carte.released_at, carte.card_faces[0].image_uris.small, carte.card_faces[1].image_uris.small, carte.card_faces[0].image_uris.normal, carte.card_faces[1].image_uris.normal, carte.type_line, carte.color_identity, carte.set_id, carte.rarity, carte.purchase_uris?.cardmarket, carte.prices.eur, type, carte.name.replace(/[^a-zA-Z0-9]/gim,"-").toLowerCase(), carte.set, layout, promo, can_be_commander]) - - } else { - const addingCardsQuery = await client.query('INSERT INTO carte(id, name, released_at, small_image, normal_image, type_line, color_identity, set_id, rarity, cardmarket_uri, price, type, sanitized_name, set_code, layout, is_promo, can_be_commander) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)', [carte.id, carte.name, carte.released_at, carte.image_uris.small, carte.image_uris.normal, carte.type_line, carte.color_identity, carte.set_id, carte.rarity, carte.purchase_uris?.cardmarket, carte.prices.eur, type, carte.name.replace(/[^a-zA-Z0-9]/gim,"-").toLowerCase(), carte.set, layout, promo, can_be_commander]) - - } - total_inserted = total_inserted + 1 - } catch (err) { - console.log(carte.uri) - console.log(carte.layout) - console.log(err) - total_skipped = total_skipped + 1 - } - } else { - const query = 'UPDATE "carte" SET "price" = $1 WHERE "id" = $2' - try { - const updateQuery = await client.query(query, [carte.prices.eur, carte.id]) - total_updated = total_updated + 1 - } catch (err) { - total_skipped = total_skipped + 1 - console.log(err) - console.log(query) - } + if(card_type.includes("creature")){ + type = "creature" + } else if (card_type.includes("planeswalker")) { + type = "planeswalker" + } else if (card_type.includes("artifact")) { + type = "artifact" + } else if (card_type.includes("instant")) { + type = "instant" + } else if (card_type.includes("enchantment")) { + type = "enchantment" + } else if (card_type.includes("sorcery")) { + type = "sorcery" + } else if (card_type.includes("land")) { + type = "land" } + + let cardObject = { + id: carte.id, + name: carte.name, + released_at: carte.released_at, + small_image: carte.card_faces[0].image_uris.small, + small_image_back: carte.card_faces[1].image_uris.small, + normal_image: carte.card_faces[0].image_uris.normal, + normal_image_back: carte.card_faces[1].image_uris.normal, + type_line: carte.type_line, + color_identity: carte.color_identity, + set_id: carte.set_id, + rarity: carte.rarity, + cardmarket_uri: carte.purchase_uris?.cardmarket, + price: carte.prices.eur, + type: type, + sanitized_name: carte.name.replace(/[^a-zA-Z0-9]/gim,"-").toLowerCase(), + set_code: carte.set, + layout: carte.layout, + is_promo: false, + can_be_commander: can_be_commander + } + if(two_faced_layouts.includes(carte.layout)) { + cardObject.small_image = carte.card_faces[0].image_uris.small + cardObject.small_image_back = carte.card_faces[1].image_uris.small + cardObject.normal_image= carte.card_faces[0].image_uris.normal + cardObject.normal_image_back= carte.card_faces[1].image_uris.normal + + } else { + cardObject.small_image = carte.image_uris.small + cardObject.normal_image= carte.image_uris.normal + } + + cardsToAdd.push(cardObject) + total_inserted = total_inserted + 1 } else { - total_skipped = total_skipped + 1 + cardsToUpdate.push({ id: carte.id, price: carte.prices.eur }) } } + console.log("Inserting cards in database...") + await db.carte.createMany({ data: cardsToAdd }) + + console.log("Updating cards in database...") + await bulkUpdate("carte", cardsToUpdate) + console.log("Un total de " + total_inserted + " cartes ont été insérées.") - console.log("Un total de " + total_skipped + " cartes ont été ignorées.") console.log("Un total de " + total_updated + " cartes ont été mises à jour.") } catch (err) { console.error(err); -} finally { - await client.end() } +const end = performance.now() +console.log(`Time taken to generate stats is ${(end - start)/1000}s.`);