globuzma/brawlset
1
0
Fork 0
Code Issues Pull requests Projects 2 Releases Packages 1 Wiki Activity

Feat: Change update database mecanism to use prisma + fetching each set data

Browse source
This commit is contained in:
zuma 2025-02-13 18:56:42 +01:00
parent ed8c0a1876
commit 46a038973a
6 changed files with 194 additions and 116 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
app/app/api/account/decks/create/route.ts
View file

@ -1,6 +1,7 @@
import { NextResponse, NextRequest } from 'next/server'
import { validateToken, decryptToken } from '@/lib/jwt'
import { db } from "@/lib/db"
import { logging } from '@/lib/logging'
interface orCardFilterProps {
sanitized_name: string
@ -72,24 +73,28 @@ export async function POST(req: NextRequest) {
// Sort cards to select non promo types first and fallback to promo cards if not found
cardsData = cardsData.sort((a,b) => +a.is_promo - +b.is_promo)
const cardsNotFound = []
let allCardFound = true
if(cardsData.findIndex(cardData => cardData.sanitized_name == commander_name) == -1){
if(cardsData.findIndex(cardData => cardData.sanitized_name.includes(commander_name)) == -1){
console.log("Not Found : " + commander_name)
allCardFound = false
cardsNotFound.push(commander_name)
}
}
cards.forEach((card: cardEntryAPIProps) => {
if(cardsData.findIndex(cardData => cardData.sanitized_name == card.sanitized_name) == -1){
if(cardsData.findIndex(cardData => cardData.sanitized_name.includes(card.sanitized_name)) == -1 ){
allCardFound = false
console.log("Not Found : " + card.sanitized_name)
console.log("Not Found : Bset = " + selected_bset + " , name = " + card.sanitized_name)
cardsNotFound.push(card.sanitized_name)
}
}
})
if(!allCardFound) {
return NextResponse.json({"message": "Some cards were not found..."},{
return NextResponse.json({"message": "Some cards were not found... " + cardsNotFound.join(", ")},{
status: 401,
});
}
@ -121,7 +126,6 @@ export async function POST(req: NextRequest) {
cards.forEach(async (card: cardEntryAPIProps) => {
const cardData_id = cardsData.findIndex(cardData => cardData.sanitized_name == card.sanitized_name) == -1 ? cardsData[cardsData.findIndex(cardData => cardData.sanitized_name.includes(card.sanitized_name))].id : cardsData[cardsData.findIndex(cardData => cardData.sanitized_name == card.sanitized_name)].id
console.log(card.sanitized_name)
await db.cartes_dans_deck.create({
data: {
amount: card.amount,
@ -155,6 +159,8 @@ export async function POST(req: NextRequest) {
}
})
logging("Deck created by " + tokenData.username + " : " + name)
return NextResponse.json({"data": deck_complete, "message": "Deck created !"},{
status: 200,
});

2
app/app/api/auth/signin/route.ts
View file

@ -3,6 +3,7 @@ import { createToken } from '@/lib/jwt'
import { createHmac } from "crypto"
import { db } from '@/lib/db'
import { logging } from '@/lib/logging'
const secret = process.env.PASSWORD_SECRET ? process.env.PASSWORD_SECRET : ""
@ -19,6 +20,7 @@ export async function POST(req: NextRequest) {
if (user !== undefined){
if(createHmac('sha256',secret).update(password).digest('hex') == user!.password) {
const token = createToken({data: {username: user!.username, admin: user!.admin, id: user!.id}, maxAge: 60*60*24*7})
logging("User " + user!.username + " has connected.")
return NextResponse.json({"JWT": token},{
status: 200,
});

17
app/app/bset/[bset]/page_content.tsx
View file

@ -25,15 +25,14 @@ export default function PageContent({bset}: PageContentProps) {
fetch('/api/json/bset/'+bset+'.json').then((res) => {
if(res.status == 200) {
res.json().then((data) => {
const limit = 100
setCommanderList(data["commander"].slice(0,limit))
setCreatureList(data["creature"].slice(0,limit))
setPlaneswalkerList(data["planeswalker"].slice(0,limit))
setSorceryList(data["sorcery"].slice(0,limit))
setInstantList(data["instant"].slice(0,limit))
setEnchantmentList(data["enchantment"].slice(0,limit))
setLandList(data["land"].slice(0,limit))
setArtifactList(data["artifact"].slice(0,limit))
setCommanderList(data["commander"])
setCreatureList(data["creature"])
setPlaneswalkerList(data["planeswalker"])
setSorceryList(data["sorcery"])
setInstantList(data["instant"])
setEnchantmentList(data["enchantment"])
setLandList(data["land"])
setArtifactList(data["artifact"])
console.log(data)
})
}

4
app/lib/logging.ts Normal file
View file

@ -0,0 +1,4 @@
export function logging(message: string) {
const date = new Date()
console.log("[" + date.getFullYear() + "/" + (date.getMonth()+1).toString().padStart(2,'0') + "/" + date.getDate().toString().padStart(2,'0') + " - " + date.getHours() + ":" + date.getMinutes() + ":" + date.getSeconds() +"] " + message)
}

6
app/tools/createJson.mjs
View file

@ -160,19 +160,19 @@ async function createJson() {
writeFileSync(import.meta.dirname + "/../data/misc/bsets.json",JSON.stringify(bsets_list_export), 'utf8')
for (const index of Object.keys(commanderData)) {
let JSONToWrite = commanderData[index].sort((a,b) => b.nbr_decks - a.nbr_decks)
let JSONToWrite = commanderData[index].sort((a,b) => b.nbr_decks - a.nbr_decks || b.percent_decks - a.percent_decks)
writeFileSync(import.meta.dirname + "/../data/commander/" + index + ".json",JSON.stringify(JSONToWrite), 'utf8')
}
for (const index of Object.keys(bset_cards_data_export)) {
let JSONToWrite = bset_cards_data_export[index]
for (const type of Object.keys(JSONToWrite)) {
JSONToWrite[type] = JSONToWrite[type].sort((a,b) => b.nbr_decks - a.nbr_decks)
JSONToWrite[type] = JSONToWrite[type].sort((a,b) => b.nbr_decks - a.nbr_decks || b.percent_decks - a.percent_decks)
}
writeFileSync(import.meta.dirname + "/../data/bset/" + index + ".json",JSON.stringify(JSONToWrite), 'utf8')
}
const end = performance.now()
console.log(`Time taken to generate stats is ${end - start}ms.`);
console.log(`Time taken to generate stats is ${(end - start)/1000}s.`);
}
createJson()

219
app/tools/updateDatabase.mjs
View file

@ -1,95 +1,149 @@
import 'dotenv/config'
import 'https'
import fs from 'fs'
import { Readable } from 'stream'
import { finished } from 'stream/promises'
import pg from 'pg'
const { Client } = pg
import { PrismaClient } from '@prisma/client'
import pkg from '@prisma/client'
const { PrismaPromise } = pkg;
import { performance } from 'perf_hooks'
console.log("Fetching latest Scryfall Bulk Data URL...")
const bulkDataApi = await fetch('https://api.scryfall.com/bulk-data')
const bulkDataApiJson = await bulkDataApi.json()
const bulkDataDownloadUrl = bulkDataApiJson.data.filter((obj) => obj.type == "default_cards")[0].download_uri
const start = performance.now()
const db = new PrismaClient()
console.log("Downloading latest Scryfall Bulk Data...")
const stream = fs.createWriteStream(import.meta.dirname + '/data/scryfall_data.json');
const { body } = await fetch(bulkDataDownloadUrl);
await finished(Readable.fromWeb(body).pipe(stream));
// Found on Github : https://github.com/prisma/prisma/discussions/19765#discussioncomment-9986300
function bulkUpdate(tableName, entries){
if (entries.length === 0) return db.$executeRawUnsafe(`SELECT 1;`);
const fields = Object.keys(entries[0]).filter((key) => key !== 'id');
const setSql = fields
.map((field) => `"${field}" = data."${field}"`)
.join(', ');
const valuesSql = entries
.map((entry) => {
const values = fields.map((field) => {
const value = entry[field];
if (typeof value === 'string') {
// Handle strings and escape single quotes
return `'${value.replace(/'/g, "''")}'`;
} else if (value instanceof Date) {
// Convert Date to ISO 8601 string format
return `'${value.toISOString()}'`;
} else if (value == null) {
// Handle null values or undefined
return `NULL`;
}
// Numbers and booleans are used as-is
return value;
});
return `('${entry.id}', ${values.join(', ')})`;
})
.join(', ');
const sql = `
UPDATE "${tableName}"
SET ${setSql}
FROM (VALUES ${valuesSql}) AS data(id, ${fields
.map((field) => `"${field}"`)
.join(', ')})
WHERE "${tableName}".id::text = data.id;
`;
return db.$executeRawUnsafe(sql);
}
console.log("Fetching latest sets list from Scryfall...")
const scryfallSets = await fetch('https://api.scryfall.com/sets');
console.log('Status Code:', scryfallSets.status);
const sets = await scryfallSets.json();
console.log("Fetching BrawlSet Set Codes...")
const bsets = await db.bset.findMany({
relationLoadStrategy: "join",
include: {
sets: true
}
})
let set_codes = []
bsets.forEach((bset) => {
bset.sets.forEach((set) => {
set_codes.push(set.code)
})
})
async function fetchApiData(url, cards) {
const apiData = await fetch(url)
const data = await apiData.json()
let res_cards = [...cards, ...data.data]
if( data.has_more) {
console.log(" fetching next page...")
return fetchApiData(data.next_page, res_cards)
} else {
return res_cards
}
}
let scryfallData = []
// Read the data from the exported fr_cards.json extracted from Scryfall Bulk Data
console.log("Reading Bulk Data...")
const fileBytes = fs.readFileSync(import.meta.dirname + '/data/scryfall_data.json')
let scryfallData = JSON.parse(fileBytes)
// Connect to postgres database
const client = new Client({
user: process.env.DATABASE_USER,
password: process.env.DATABASE_PASSWORD,
host: process.env.DATABASE_HOST,
port: process.env.DATABASE_PORT,
database: process.env.DATABASE_DB
})
await client.connect()
let nbr_sets = set_codes.length
for(const [index, code] of set_codes.entries()) {
console.log("fetching " + code + "... " + index + "/" + nbr_sets)
const setCardsData = await fetchApiData("https://api.scryfall.com/cards/search?q=(game%3Apaper)+set%3A" + code, [])
scryfallData = [...scryfallData, ...setCardsData]
}
const two_faced_layouts = ["transform","modal_dfc","double_faced_token","reversible_card"]
const prohibited_frame_effects = ["extendedart", "showcase", "upsidedowndf", "waxingandwaningmoondfc", "shatteredglass", "convertdfc", "originpwdfc", "draft", "etched", "enchantment","inverted"]
console.log("Starting updating database...")
const total_cards = scryfallData.length
console.log("Creating objects to update database... (" + total_cards + " cards found)" )
try {
const setRes = await client.query('SELECT id FROM set')
const preUpdateSetRows = setRes.rows
const setsRes = await db.set.findMany()
let preUpdateSetIds = []
preUpdateSetRows.forEach(element => {
setsRes.forEach(element => {
preUpdateSetIds.push(element.id)
});
for (const set of sets.data) {
if(!preUpdateSetIds.includes(set.id)){
await client.query('INSERT INTO set(id, name_en, sanitized_name, code, set_type, released_at, icon_svg_uri) VALUES($1, $2, $3, $4, $5, $6, $7)', [set.id, set.name, set.name.replace(/[^a-zA-Z0-9]/gim,"-").toLowerCase(), set.code, set.set_type, set.released_at, set.icon_svg_uri])
await db.set.create({
data: {
id: set.id,
name_en: set.name,
sanitized_name: set.name.replace(/[^a-zA-Z0-9]/gim,"-").toLowerCase(),
code: set.code,
set_type: set.set_type,
released_at: set.released_at,
icon_svg_uri: set.icon_svg_uri
}
})
}
}
// Select already imported cards in database
const cardsRes = await client.query('SELECT id FROM carte')
const preUpdateCardsRows = cardsRes.rows
const cardsRes = await db.carte.findMany()
let preUpdateCardsIds = []
preUpdateCardsRows.forEach(element => {
cardsRes.forEach(element => {
preUpdateCardsIds.push(element.id)
});
// Define counter for logging
let total_inserted = 0
let total_skipped = 0
let total_updated = 0
const total_cards = scryfallData.length
const cardsToAdd = []
const cardsToUpdate = []
// For each card check if we need to upload it to the database
for (const carte of scryfallData) {
const total_processed = total_skipped + total_updated + total_inserted
if ((total_processed) % 1000 == 0) {
console.log(total_processed + "/" + total_cards)
}
if(carte.legalities.commander != "not_legal" && carte.border_color != "borderless" && !carte.full_art && !carte.textless && carte.cardmarket_id && (carte.frame_effects == undefined || prohibited_frame_effects.every((frame_effect) => !carte.frame_effects.includes(frame_effect)))) {
if(!preUpdateCardsIds.includes(carte.id)){
let type = ""
const layout = carte.layout
const card_type = (carte.type_line == undefined) ? carte.card_faces[0].type_line.toLowerCase() : carte.type_line.toLowerCase()
let promo = (carte.promo_types == undefined) ? false : true
let can_be_commander = (card_type.includes("legendary") && (card_type.includes("creature") || card_type.includes("planeswalker"))) ? true : false
if(card_type.includes("creature")){
type = "creature"
} else if (card_type.includes("planeswalker")) {
@ -106,41 +160,54 @@ try {
type = "land"
}
try {
if(two_faced_layouts.includes(layout)) {
const addingCardsQuery = await client.query('INSERT INTO carte(id, name, released_at, small_image, small_image_back, normal_image, normal_image_back, type_line, color_identity, set_id, rarity, cardmarket_uri, price, type, sanitized_name, set_code, layout, is_promo, can_be_commander) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19)', [carte.id, carte.name, carte.released_at, carte.card_faces[0].image_uris.small, carte.card_faces[1].image_uris.small, carte.card_faces[0].image_uris.normal, carte.card_faces[1].image_uris.normal, carte.type_line, carte.color_identity, carte.set_id, carte.rarity, carte.purchase_uris?.cardmarket, carte.prices.eur, type, carte.name.replace(/[^a-zA-Z0-9]/gim,"-").toLowerCase(), carte.set, layout, promo, can_be_commander])
let cardObject = {
id: carte.id,
name: carte.name,
released_at: carte.released_at,
small_image: carte.card_faces[0].image_uris.small,
small_image_back: carte.card_faces[1].image_uris.small,
normal_image: carte.card_faces[0].image_uris.normal,
normal_image_back: carte.card_faces[1].image_uris.normal,
type_line: carte.type_line,
color_identity: carte.color_identity,
set_id: carte.set_id,
rarity: carte.rarity,
cardmarket_uri: carte.purchase_uris?.cardmarket,
price: carte.prices.eur,
type: type,
sanitized_name: carte.name.replace(/[^a-zA-Z0-9]/gim,"-").toLowerCase(),
set_code: carte.set,
layout: carte.layout,
is_promo: false,
can_be_commander: can_be_commander
}
if(two_faced_layouts.includes(carte.layout)) {
cardObject.small_image = carte.card_faces[0].image_uris.small
cardObject.small_image_back = carte.card_faces[1].image_uris.small
cardObject.normal_image= carte.card_faces[0].image_uris.normal
cardObject.normal_image_back= carte.card_faces[1].image_uris.normal
} else {
const addingCardsQuery = await client.query('INSERT INTO carte(id, name, released_at, small_image, normal_image, type_line, color_identity, set_id, rarity, cardmarket_uri, price, type, sanitized_name, set_code, layout, is_promo, can_be_commander) VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)', [carte.id, carte.name, carte.released_at, carte.image_uris.small, carte.image_uris.normal, carte.type_line, carte.color_identity, carte.set_id, carte.rarity, carte.purchase_uris?.cardmarket, carte.prices.eur, type, carte.name.replace(/[^a-zA-Z0-9]/gim,"-").toLowerCase(), carte.set, layout, promo, can_be_commander])
cardObject.small_image = carte.image_uris.small
cardObject.normal_image= carte.image_uris.normal
}
cardsToAdd.push(cardObject)
total_inserted = total_inserted + 1
} catch (err) {
console.log(carte.uri)
console.log(carte.layout)
console.log(err)
total_skipped = total_skipped + 1
}
} else {
const query = 'UPDATE "carte" SET "price" = $1 WHERE "id" = $2'
try {
const updateQuery = await client.query(query, [carte.prices.eur, carte.id])
total_updated = total_updated + 1
} catch (err) {
total_skipped = total_skipped + 1
console.log(err)
console.log(query)
}
}
} else {
total_skipped = total_skipped + 1
cardsToUpdate.push({ id: carte.id, price: carte.prices.eur })
}
}
console.log("Inserting cards in database...")
await db.carte.createMany({ data: cardsToAdd })
console.log("Updating cards in database...")
await bulkUpdate("carte", cardsToUpdate)
console.log("Un total de " + total_inserted + " cartes ont été insérées.")
console.log("Un total de " + total_skipped + " cartes ont été ignorées.")
console.log("Un total de " + total_updated + " cartes ont été mises à jour.")
} catch (err) {
console.error(err);
} finally {
await client.end()
}
const end = performance.now()
console.log(`Time taken to generate stats is ${(end - start)/1000}s.`);