feat: balayage de re-extraction de toute la base

Nouvelle page /re-extraction, accessible depuis la liste des documents : elle
rejoue l'extraction de chaque PDF stocke, document par document, avec une
progression et une estimation du temps restant (une extraction prend plusieurs
secondes) et un bouton d'arret. Rien n'est ecrit en base pendant le balayage.

Les documents dont l'extraction change sont listes avec le nombre de
differences par section, deja coches ; les inchanges sont replies et les
documents sans PDF ou en erreur isoles. Le detail depliable reutilise
l'affichage de diff de la re-extraction unitaire (extrait dans
ExtractionDiffDetails). Un seul bouton enregistre toute la selection.

Les tags de depenses sont reportes par signature d'operation et non par index :
une operation ajoutee ou retiree par le nouveau parser ne decale plus les
categories. Ce qui n'est pas reapparie recoit une prediction, modifiable dans le
detail avant enregistrement.

L'edition d'un document et l'application du balayage passent par
PUT /api/documents/{id}, qui vise le document par son ID.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-26 05:17:49 +02:00
parent 6f2530014f
commit 5f3a26e7d7
8 changed files with 784 additions and 72 deletions

View File

@@ -10,6 +10,20 @@
{{ documents.length }} document{{ documents.length > 1 ? 's' : '' }} en base
</p>
</div>
<div class="flex items-center gap-3">
<!-- Re-extraction de toute la base -->
<router-link
to="/re-extraction"
class="btn btn-secondary"
title="Rejouer l'extraction de tous les PDF stockes"
>
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
d="M4 4v5h.582m15.356 2A8.001 8.001 0 004.582 9m0 0H9m11 11v-5h-.581m0 0a8.003 8.003 0 01-15.357-2m15.357 2H15" />
</svg>
Re-extraire la base
</router-link>
<!-- Import button -->
<label class="btn btn-primary cursor-pointer">
@@ -24,6 +38,7 @@
@change="onFileSelect"
/>
</label>
</div>
</div>
<!-- Loading state -->

View File

@@ -262,16 +262,17 @@ async function handleSave(depensesTags, shouldOverwrite) {
saveError.value = null
try {
const response = await fetch('/api/save', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
// Mise à jour ciblée par ID : si la nouvelle extraction corrige la référence
// ou la date, c'est bien ce document qui est mis à jour, pas un doublon.
const response = await fetch(`/api/documents/${documentId}`, {
method: 'PUT',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
source_file: extractedData.value.source_file,
data: extractedData.value.data,
depenses_tags: depensesTags,
overwrite: true // Toujours true en mode édition
depenses_tags: depensesTags
})
})

View File

@@ -0,0 +1,578 @@
<template>
<div class="flex-1 overflow-auto bg-gray-800">
<div class="max-w-6xl mx-auto p-6 space-y-6 pb-28">
<!-- Header -->
<div class="flex items-start justify-between gap-6">
<div>
<h1 class="text-xl font-semibold text-white">-extraction de la base</h1>
<p class="text-sm text-gray-400 mt-1 max-w-2xl">
Rejoue l'extraction de chaque PDF stocké après une amélioration des parsers.
Rien n'est écrit en base tant que vous n'avez pas validé : les documents dont
l'extraction change sont listés ci-dessous, à appliquer en une fois.
</p>
</div>
<div class="flex items-center gap-3 flex-shrink-0">
<button
v-if="!isScanning"
@click="startScan"
:disabled="isLoading || isApplying || documents.length === 0"
class="px-4 py-2 bg-blue-600 text-white rounded-lg hover:bg-blue-700 transition-colors disabled:opacity-50 flex items-center gap-2"
>
<svg class="w-5 h-5" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
d="M4 4v5h.582m15.356 2A8.001 8.001 0 004.582 9m0 0H9m11 11v-5h-.581m0 0a8.003 8.003 0 01-15.357-2m15.357 2H15" />
</svg>
{{ hasScanned ? 'Relancer le balayage' : 'Lancer le balayage' }}
</button>
<button
v-else
@click="stopScan"
class="px-4 py-2 bg-gray-700 text-white rounded-lg hover:bg-gray-600 transition-colors"
>
Arrêter
</button>
</div>
</div>
<!-- Progression du balayage -->
<div v-if="isScanning || hasScanned" class="bg-gray-900 rounded-lg border border-gray-700 p-4 space-y-3">
<div class="flex items-center justify-between text-sm">
<span class="text-white">
<template v-if="isScanning">
Balayage en cours {{ scanned }} / {{ documents.length }}
</template>
<template v-else>
Balayage terminé {{ scanned }} / {{ documents.length }} document{{ documents.length > 1 ? 's' : '' }}
</template>
</span>
<span v-if="isScanning" class="text-gray-400">
{{ remainingLabel }}
</span>
</div>
<div class="h-2 bg-gray-700 rounded overflow-hidden">
<div
class="h-full bg-blue-500 transition-all duration-300"
:style="{ width: progressPercent + '%' }"
></div>
</div>
<div class="flex flex-wrap items-center gap-4 text-xs">
<span class="text-amber-400">{{ counts.modifie }} à réenregistrer</span>
<span class="text-gray-400">{{ counts.identique }} inchangé{{ counts.identique > 1 ? 's' : '' }}</span>
<span v-if="counts.applique" class="text-green-400">{{ counts.applique }} appliqué{{ counts.applique > 1 ? 's' : '' }}</span>
<span v-if="counts.sans_pdf" class="text-gray-500">{{ counts.sans_pdf }} sans PDF</span>
<span v-if="counts.erreur || counts.echec" class="text-red-400">
{{ counts.erreur + counts.echec }} en erreur
</span>
</div>
</div>
<!-- Chargement initial -->
<div v-if="isLoading" class="bg-gray-900 rounded-lg p-12 text-center">
<div class="inline-block animate-spin rounded-full h-8 w-8 border-2 border-gray-600 border-t-blue-400"></div>
<p class="text-gray-400 mt-4">Chargement des documents...</p>
</div>
<!-- Invitation initiale -->
<div v-else-if="!hasScanned && !isScanning" class="bg-gray-900 rounded-lg border border-gray-700 p-8 text-center">
<p class="text-gray-300">
{{ documents.length }} document{{ documents.length > 1 ? 's' : '' }} en base,
dont {{ documents.filter((d) => d.has_pdf).length }} avec PDF stocké.
</p>
<p class="text-gray-500 text-sm mt-2">
Comptez environ {{ SECONDS_PER_DOC }} s par document, soit
{{ formatDuration(documents.filter((d) => d.has_pdf).length * SECONDS_PER_DOC) }} au total.
</p>
</div>
<!-- Documents modifiés -->
<div v-if="changedItems.length" class="bg-gray-900 rounded-lg border border-gray-700 overflow-hidden">
<div class="flex items-center justify-between px-4 py-3 bg-gray-800 border-b border-gray-700">
<label class="flex items-center gap-2 text-sm text-white cursor-pointer">
<input
type="checkbox"
:checked="allSelected"
:indeterminate.prop="someSelected && !allSelected"
@change="toggleAll"
class="w-4 h-4 rounded"
/>
Extraction modifiée ({{ changedItems.length }})
</label>
<span class="text-xs text-gray-400">{{ selectedItems.length }} sélectionné{{ selectedItems.length > 1 ? 's' : '' }}</span>
</div>
<div class="divide-y divide-gray-800">
<div v-for="item in changedItems" :key="item.id">
<!-- Ligne -->
<div class="flex items-center gap-3 px-4 py-3 hover:bg-gray-800/50 transition-colors">
<input
type="checkbox"
v-model="item.selected"
:disabled="isApplying"
class="w-4 h-4 rounded flex-shrink-0"
/>
<div class="flex-1 min-w-0">
<div class="flex items-center gap-2">
<span class="text-sm font-medium text-white font-mono">{{ item.reference }}</span>
<span class="text-xs text-gray-500">{{ formatDate(item.date) }}</span>
</div>
<div class="text-xs text-gray-500 truncate">{{ item.immeuble_adresse || '-' }}</div>
</div>
<!-- Résumé des différences -->
<div class="flex items-center gap-1.5 flex-shrink-0">
<span v-if="item.diff.summary.metadata" class="px-2 py-0.5 text-xs rounded bg-purple-500/20 text-purple-300">
{{ item.diff.summary.metadata }} métadonnée{{ item.diff.summary.metadata > 1 ? 's' : '' }}
</span>
<span v-if="item.diff.summary.locataires" class="px-2 py-0.5 text-xs rounded bg-blue-500/20 text-blue-300">
{{ item.diff.summary.locataires }} locataire{{ item.diff.summary.locataires > 1 ? 's' : '' }}
</span>
<span v-if="item.diff.summary.operations" class="px-2 py-0.5 text-xs rounded bg-amber-500/20 text-amber-300">
{{ item.diff.summary.operations }} opération{{ item.diff.summary.operations > 1 ? 's' : '' }}
</span>
</div>
<!-- État des tags -->
<div class="w-44 flex-shrink-0 text-xs text-right">
<span class="text-gray-400">{{ item.tagStats.reported }} tag{{ item.tagStats.reported > 1 ? 's' : '' }} reporté{{ item.tagStats.reported > 1 ? 's' : '' }}</span>
<div v-if="item.tagStats.predicted" class="text-blue-300">{{ item.tagStats.predicted }} prédit{{ item.tagStats.predicted > 1 ? 's' : '' }}</div>
<div v-if="item.tagStats.untagged" class="text-amber-300">{{ item.tagStats.untagged }} sans tag</div>
</div>
<!-- Actions -->
<div class="flex items-center gap-2 flex-shrink-0">
<span v-if="item.status === 'applique'" class="px-2 py-1 text-xs rounded bg-green-500/20 text-green-400">
Appliqué
</span>
<span v-else-if="item.status === 'echec'" class="px-2 py-1 text-xs rounded bg-red-500/20 text-red-400" :title="item.error">
Échec
</span>
<button
@click="toggleDetail(item.id)"
class="px-3 py-1.5 text-xs rounded bg-gray-700 text-gray-300 hover:bg-gray-600 transition-colors"
>
{{ openDetails[item.id] ? 'Masquer' : 'Détail' }}
</button>
<a
:href="`/api/documents/${item.id}/pdf`"
target="_blank"
class="px-2 py-1.5 text-xs rounded bg-red-500/20 text-red-400 hover:bg-red-500/30 transition-colors"
title="Ouvrir le PDF"
>
PDF
</a>
</div>
</div>
<!-- Détail : différences + tags à confirmer -->
<div v-if="openDetails[item.id]" class="bg-amber-50 border-t border-amber-300 px-4 py-3 space-y-4">
<ExtractionDiffDetails :diff="item.diff" />
<div v-if="item.tagsToReview.length">
<div class="font-semibold text-amber-800 text-xs mb-2">
Tags à confirmer ({{ item.tagStats.reported }} reporté{{ item.tagStats.reported > 1 ? 's' : '' }} automatiquement)
</div>
<div class="space-y-2">
<div
v-for="opIndex in item.tagsToReview"
:key="opIndex"
class="flex items-center gap-3 bg-white/70 rounded p-2"
>
<div class="flex-1 min-w-0 text-xs">
<div class="font-medium text-gray-800 truncate">
{{ item.newData.recapitulatif_operations[opIndex].fournisseur || 'Fournisseur inconnu' }}
</div>
<div class="text-gray-500 truncate">
{{ item.newData.recapitulatif_operations[opIndex].sous_categorie
|| item.newData.recapitulatif_operations[opIndex].description || '-' }}
· {{ formatAmount(item.newData.recapitulatif_operations[opIndex].montants?.debit) }}
</div>
<div v-if="item.tags[opIndex]?.origin === 'predit'" class="text-blue-600 mt-0.5">
Prédit ({{ item.tags[opIndex].confidence }}%) {{ item.tags[opIndex].reason }}
</div>
</div>
<div class="w-56 flex-shrink-0">
<TagAutocomplete
:model-value="item.tags[opIndex]?.tag_id ?? null"
:options="availableTags"
:has-prediction="item.tags[opIndex]?.origin === 'predit'"
:is-empty="!item.tags[opIndex]?.tag_id"
@update:modelValue="setTag(item, opIndex, $event)"
/>
</div>
</div>
</div>
</div>
<p v-else class="text-xs text-amber-800">
Les {{ item.tagStats.reported }} tag{{ item.tagStats.reported > 1 ? 's' : '' }} du document
{{ item.tagStats.reported > 1 ? 'sont reportés' : 'est reporté' }} sur la nouvelle extraction.
</p>
</div>
</div>
</div>
</div>
<!-- Documents en erreur -->
<div v-if="problemItems.length" class="bg-gray-900 rounded-lg border border-gray-700 overflow-hidden">
<div class="px-4 py-3 bg-gray-800 border-b border-gray-700 text-sm text-white">
Non -extraits ({{ problemItems.length }})
</div>
<div class="divide-y divide-gray-800">
<div v-for="item in problemItems" :key="item.id" class="flex items-center gap-3 px-4 py-2.5">
<div class="flex-1 min-w-0">
<span class="text-sm font-mono text-white">{{ item.reference }}</span>
<span class="text-xs text-gray-500 ml-2">{{ formatDate(item.date) }}</span>
</div>
<span class="text-xs" :class="item.status === 'sans_pdf' ? 'text-gray-500' : 'text-red-400'">
{{ item.status === 'sans_pdf' ? 'PDF non stocké' : item.error }}
</span>
</div>
</div>
</div>
<!-- Documents inchangés -->
<div v-if="identicalItems.length" class="bg-gray-900 rounded-lg border border-gray-700 overflow-hidden">
<button
@click="showIdentical = !showIdentical"
class="w-full flex items-center justify-between px-4 py-3 bg-gray-800 text-sm text-gray-300 hover:text-white transition-colors"
>
<span>{{ identicalItems.length }} document{{ identicalItems.length > 1 ? 's' : '' }} inchangé{{ identicalItems.length > 1 ? 's' : '' }}</span>
<span class="text-xs">{{ showIdentical ? 'Masquer' : 'Voir' }}</span>
</button>
<div v-if="showIdentical" class="divide-y divide-gray-800">
<div v-for="item in identicalItems" :key="item.id" class="flex items-center gap-3 px-4 py-2 text-xs">
<span class="font-mono text-gray-300">{{ item.reference }}</span>
<span class="text-gray-500">{{ formatDate(item.date) }}</span>
<span class="text-gray-600 truncate">{{ item.immeuble_adresse || '-' }}</span>
</div>
</div>
</div>
</div>
<!-- Barre d'application -->
<div
v-if="changedItems.length"
class="fixed bottom-0 left-0 right-0 bg-gray-900 border-t border-gray-700 px-6 py-3"
>
<div class="max-w-6xl mx-auto flex items-center justify-between gap-4">
<div class="text-sm" :class="applyError && !isApplying ? 'text-red-400' : 'text-gray-400'">
<template v-if="isApplying">
Enregistrement {{ applyDone }} / {{ applyTotal }}...
</template>
<template v-else-if="applyError">
{{ applyError }}
</template>
<template v-else>
Les documents sélectionnés seront réenregistrés avec la nouvelle extraction.
</template>
</div>
<button
@click="applySelection"
:disabled="isApplying || isScanning || selectedItems.length === 0"
class="px-5 py-2 bg-green-600 text-white rounded-lg hover:bg-green-700 transition-colors font-medium disabled:opacity-50 disabled:cursor-not-allowed"
>
Appliquer la sélection ({{ selectedItems.length }})
</button>
</div>
</div>
</div>
</template>
<script setup>
import { computed, onMounted, onBeforeUnmount, reactive, ref } from 'vue'
import ExtractionDiffDetails from '../components/ExtractionDiffDetails.vue'
import TagAutocomplete from '../components/TagAutocomplete.vue'
import { computeExtractionDiff } from '../utils/diffExtraction.js'
import { fillPredictions, remapTags, toDepensesTags } from '../utils/tagRemap.js'
// Ordre de grandeur mesuré d'une extraction, avant toute mesure réelle.
const SECONDS_PER_DOC = 7
const documents = ref([])
const items = ref([])
const availableTags = ref([])
const openDetails = reactive({})
const showIdentical = ref(false)
const isLoading = ref(true)
const isScanning = ref(false)
const hasScanned = ref(false)
const scanned = ref(0)
const durations = ref([])
const isApplying = ref(false)
const applyDone = ref(0)
const applyTotal = ref(0)
const applyError = ref(null)
let abortScan = false
let controller = null
const changedItems = computed(() =>
items.value.filter((i) => ['modifie', 'applique', 'echec'].includes(i.status))
)
const identicalItems = computed(() => items.value.filter((i) => i.status === 'identique'))
const problemItems = computed(() =>
items.value.filter((i) => ['sans_pdf', 'erreur'].includes(i.status))
)
const selectedItems = computed(() => changedItems.value.filter((i) => i.selected))
const allSelected = computed(
() => changedItems.value.length > 0 && selectedItems.value.length === changedItems.value.length
)
const someSelected = computed(() => selectedItems.value.length > 0)
const counts = computed(() => {
const c = { modifie: 0, identique: 0, sans_pdf: 0, erreur: 0, applique: 0, echec: 0 }
for (const i of items.value) if (i.status in c) c[i.status]++
return c
})
const progressPercent = computed(() => {
if (!documents.value.length) return 0
return Math.round((scanned.value / documents.value.length) * 100)
})
const remainingLabel = computed(() => {
const left = documents.value.length - scanned.value
if (left <= 0) return ''
const avg = durations.value.length
? durations.value.reduce((s, d) => s + d, 0) / durations.value.length
: SECONDS_PER_DOC
return `${formatDuration(left * avg)} restant`
})
onMounted(async () => {
await Promise.all([loadDocuments(), loadTags()])
isLoading.value = false
})
onBeforeUnmount(() => {
abortScan = true
controller?.abort()
})
async function loadDocuments() {
try {
const response = await fetch('/api/documents?limit=1000')
if (!response.ok) throw new Error('Chargement des documents impossible')
documents.value = await response.json()
resetItems()
} catch (err) {
console.error('Erreur de chargement des documents:', err)
}
}
async function loadTags() {
try {
const response = await fetch('/api/tags')
if (response.ok) availableTags.value = await response.json()
} catch (err) {
console.error('Erreur de chargement des tags:', err)
}
}
function resetItems() {
items.value = documents.value.map((doc) => ({
id: doc.id,
reference: doc.reference,
date: doc.date,
source_file: doc.source_file,
immeuble_adresse: doc.immeuble_adresse,
has_pdf: doc.has_pdf,
status: 'attente',
diff: null,
newData: null,
tags: {},
tagStats: { reported: 0, predicted: 0, untagged: 0 },
tagsToReview: [],
selected: false,
error: null,
}))
for (const key of Object.keys(openDetails)) delete openDetails[key]
}
async function startScan() {
resetItems()
scanned.value = 0
durations.value = []
applyError.value = null
abortScan = false
isScanning.value = true
hasScanned.value = true
for (const item of items.value) {
if (abortScan) break
await scanOne(item)
scanned.value++
}
isScanning.value = false
}
async function scanOne(item) {
if (!item.has_pdf) {
item.status = 'sans_pdf'
return
}
const startedAt = Date.now()
controller = new AbortController()
try {
const response = await fetch(`/api/documents/${item.id}/re-extract`, {
method: 'POST',
signal: controller.signal,
})
const result = await response.json()
if (!response.ok) throw new Error(result.detail || 'Échec de la ré-extraction')
const diff = computeExtractionDiff(result.previous_data, result.re_extracted_data)
item.newData = result.re_extracted_data
item.diff = diff
if (diff.summary.total === 0) {
item.status = 'identique'
return
}
await prepareTags(item, result)
item.status = 'modifie'
item.selected = true
} catch (err) {
if (err.name === 'AbortError') return
console.error(`Ré-extraction du document ${item.id}:`, err)
item.status = 'erreur'
item.error = err.message || 'Erreur inconnue'
} finally {
durations.value.push((Date.now() - startedAt) / 1000)
}
}
// Reporte les tags actuels sur la nouvelle extraction, puis complète par
// prédiction ce qui n'a pas pu être réapparié.
async function prepareTags(item, result) {
const previousOps = result.previous_data?.recapitulatif_operations || []
const newOps = result.re_extracted_data?.recapitulatif_operations || []
const { tags, reported, missing } = remapTags(previousOps, result.depenses_tags, newOps)
const predicted = await fillPredictions(tags, newOps, missing)
item.tags = tags
item.tagStats = {
reported,
predicted,
untagged: missing.length - predicted,
}
// Seules les opérations non reportées à l'identique demandent une confirmation.
item.tagsToReview = missing
}
function setTag(item, opIndex, tagId) {
if (tagId == null) {
delete item.tags[opIndex]
} else {
const tag = availableTags.value.find((t) => t.id === tagId)
item.tags[opIndex] = { tag_id: tagId, tag_nom: tag?.nom || null, origin: 'manuel' }
}
item.tags = { ...item.tags }
refreshTagStats(item)
}
function refreshTagStats(item) {
let predicted = 0
for (const opIndex of item.tagsToReview) {
if (item.tags[opIndex]?.tag_id != null) predicted++
}
item.tagStats = {
...item.tagStats,
predicted,
untagged: item.tagsToReview.length - predicted,
}
}
function stopScan() {
abortScan = true
controller?.abort()
isScanning.value = false
}
function toggleAll(event) {
const value = event.target.checked
for (const item of changedItems.value) item.selected = value
}
function toggleDetail(id) {
openDetails[id] = !openDetails[id]
}
async function applySelection() {
const targets = selectedItems.value
if (!targets.length) return
const confirmed = confirm(
`Réenregistrer ${targets.length} document${targets.length > 1 ? 's' : ''} avec la nouvelle extraction ?\n\n` +
'Les données actuelles de ces documents seront remplacées.'
)
if (!confirmed) return
isApplying.value = true
applyError.value = null
applyDone.value = 0
applyTotal.value = targets.length
let failures = 0
for (const item of targets) {
try {
const response = await fetch(`/api/documents/${item.id}`, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
source_file: item.source_file,
data: item.newData,
depenses_tags: toDepensesTags(item.tags),
}),
})
const result = await response.json()
if (!response.ok) throw new Error(result.detail || 'Échec de l\'enregistrement')
if (!result.success) throw new Error(result.message || 'Échec de l\'enregistrement')
item.status = 'applique'
item.selected = false
} catch (err) {
console.error(`Enregistrement du document ${item.id}:`, err)
item.status = 'echec'
item.error = err.message || 'Erreur inconnue'
failures++
} finally {
applyDone.value++
}
}
isApplying.value = false
if (failures) {
applyError.value = `${failures} document${failures > 1 ? 's' : ''} n'${failures > 1 ? 'ont' : 'a'} pas pu être enregistré${failures > 1 ? 's' : ''}`
}
}
function formatDate(dateStr) {
if (!dateStr) return '-'
const [year, month, day] = dateStr.split('-')
return `${day}/${month}/${year}`
}
function formatAmount(amount) {
if (amount == null) return '-'
return new Intl.NumberFormat('fr-FR', { style: 'currency', currency: 'EUR' }).format(amount)
}
function formatDuration(seconds) {
const total = Math.max(0, Math.round(seconds))
if (total < 60) return `${total} s`
const minutes = Math.round(total / 60)
return `${minutes} min`
}
</script>