TheAnachronism/docspell
1
0
Fork 0
mirror of https://github.com/TheAnachronism/docspell.git synced 2025-06-22 02:18:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Migrate file ids

Browse Source 
Issue: #1379
This commit is contained in:
eikek
2022-02-13 19:55:02 +01:00
parent 553b1fa249
commit 55700992c2
3 changed files with 448 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

150
modules/store/src/main/resources/db/migration/h2/V1.33.0__reorganize_file_ids.sql Normal file
View File

@ -0,0 +1,150 @@
drop table if exists file_migration_temp;
create table file_migration_temp (
id bigserial primary key,
original_file varchar(254) not null unique,
cid varchar(254) not null,
category varchar(254) not null,
new_file varchar(254) not null unique
);
-- Source files
insert into file_migration_temp (original_file, cid, category, new_file)
select
rs.file_id as original_file,
i.cid,
'attachmentsource' as category,
i.cid || '/attachmentsource/' || rs.file_id as new_file
from attachment_source rs
inner join attachment ra on rs.id = ra.attachid
inner join item i on ra.itemid = i.itemid
;
-- Archive files
insert into file_migration_temp (original_file, cid, category, new_file)
select distinct
rs.file_id as original_file,
i.cid,
'attachmentsource' as category,
i.cid || '/attachmentsource/' || rs.file_id as new_file
from attachment_archive rs
inner join attachment ra on rs.id = ra.attachid
inner join item i on ra.itemid = i.itemid
;
-- Converted files
insert into file_migration_temp (original_file, cid, category, new_file)
select
ra.filemetaid as original_file,
i.cid,
'attachmentconvert' as category,
i.cid || '/attachmentconvert/' || ra.filemetaid as new_file
from attachment_source rs
inner join attachment ra on rs.id = ra.attachid
inner join item i on ra.itemid = i.itemid
where rs.file_id <> ra.filemetaid
;
-- Preview image
insert into file_migration_temp (original_file, cid, category, new_file)
select
ap.file_id as original_file,
i.cid,
'previewimage' as category,
i.cid || '/previewimage/' || ap.file_id as new_file
from attachment_preview ap
inner join attachment ra on ra.attachid = ap.id
inner join item i on i.itemid = ra.itemid
order by id
;
-- classifier
insert into file_migration_temp (original_file, cid, category, new_file)
select
file_id as original_file,
cid,
'classifier' as category,
cid || '/classifier/' || file_id as new_file
from classifier_model
;
-- save obsolete/orphaned files
drop table if exists obsolete_files;
create table obsolete_files(
file_id varchar(254) not null,
mimetype varchar(254) not null,
length bigint not null,
checksum varchar(254) not null,
created timestamp not null
);
with
missing_ids as (
select file_id from filemeta
except
select original_file as file_id from file_migration_temp)
insert into obsolete_files (file_id, mimetype, length, checksum, created)
select file_id, mimetype, length, checksum, created from filemeta
where file_id in (select file_id from missing_ids)
;
-- duplicate each filemeta with the new id
insert into filemeta (file_id, mimetype, length, checksum, created)
select mm.new_file, fm.mimetype, fm.length, fm.checksum, fm.created
from file_migration_temp mm
inner join filemeta fm on fm.file_id = mm.original_file
;
-- update each reference to the new id
update attachment_source
set file_id = (select new_file
from file_migration_temp
where original_file = file_id and attachment_source.id is not null)
;
update attachment
set filemetaid = (select new_file
from file_migration_temp
where original_file = filemetaid and attachment.attachid is not null)
;
update attachment_archive
set file_id = (select new_file
from file_migration_temp
where original_file = file_id and attachment_archive.id is not null)
;
update attachment_preview
set file_id = (select new_file
from file_migration_temp
where original_file = file_id and attachment_preview.id is not null)
;
update classifier_model
set file_id = (select new_file
from file_migration_temp
where original_file = file_id and classifier_model.id is not null)
;
-- delete old filemeta and filechunk rows
delete from filemeta
where file_id in (select original_file from file_migration_temp);
delete from filemeta
where file_id in (select file_id from obsolete_files);
delete from filechunk
where file_id in (select file_id from obsolete_files);
-- update chunks
update filechunk
set file_id = (select new_file
from file_migration_temp
where original_file = file_id and filechunk.file_id is not null)
;
-- drop temp table
drop table file_migration_temp;
drop table obsolete_files;

148
modules/store/src/main/resources/db/migration/mariadb/V1.33.0__reorganize_file_ids.sql Normal file
View File

@ -0,0 +1,148 @@
drop table if exists file_migration_temp;
create table file_migration_temp (
id bigint auto_increment primary key,
original_file varchar(254) not null unique,
cid varchar(254) not null,
category varchar(254) not null,
new_file varchar(254) not null unique
);
-- Source files
insert into file_migration_temp (original_file, cid, category, new_file)
select
rs.file_id as original_file,
i.cid,
'attachmentsource' as category,
concat(i.cid, '/attachmentsource/', rs.file_id) as new_file
from attachment_source rs
inner join attachment ra on rs.id = ra.attachid
inner join item i on ra.itemid = i.itemid
;
-- Archive files
insert into file_migration_temp (original_file, cid, category, new_file)
select distinct
rs.file_id as original_file,
i.cid,
'attachmentsource' as category,
concat(i.cid, '/attachmentsource/', rs.file_id) as new_file
from attachment_archive rs
inner join attachment ra on rs.id = ra.attachid
inner join item i on ra.itemid = i.itemid
;
-- Converted files
insert into file_migration_temp (original_file, cid, category, new_file)
select
ra.filemetaid as original_file,
i.cid,
'attachmentconvert' as category,
concat(i.cid, '/attachmentconvert/', ra.filemetaid) as new_file
from attachment_source rs
inner join attachment ra on rs.id = ra.attachid
inner join item i on ra.itemid = i.itemid
where rs.file_id <> ra.filemetaid
;
-- Preview image
insert into file_migration_temp (original_file, cid, category, new_file)
select
ap.file_id as original_file,
i.cid,
'previewimage' as category,
concat(i.cid, '/previewimage/', ap.file_id) as new_file
from attachment_preview ap
inner join attachment ra on ra.attachid = ap.id
inner join item i on i.itemid = ra.itemid
order by id
;
-- classifier
insert into file_migration_temp (original_file, cid, category, new_file)
select
file_id as original_file,
cid,
'classifier' as category,
concat(cid, '/classifier/', file_id) as new_file
from classifier_model
;
-- save obsolete/orphaned files
drop table if exists obsolete_files;
create table obsolete_files(
file_id varchar(254) not null,
mimetype varchar(254) not null,
length bigint not null,
checksum varchar(254) not null,
created timestamp not null
);
insert into obsolete_files (file_id, mimetype, length, checksum, created)
select file_id, mimetype, length, checksum, created from filemeta
where file_id in (
select file_id from filemeta
except
select original_file as file_id from file_migration_temp
);
-- duplicate each filemeta with the new id
insert into filemeta (file_id, mimetype, length, checksum, created)
select mm.new_file, fm.mimetype, fm.length, fm.checksum, fm.created
from file_migration_temp mm
inner join filemeta fm on fm.file_id = mm.original_file
;
-- update each reference to the new id
update attachment_source
set file_id = (select new_file
from file_migration_temp
where original_file = file_id and attachment_source.id is not null)
;
update attachment
set filemetaid = (select new_file
from file_migration_temp
where original_file = filemetaid and attachment.attachid is not null)
;
update attachment_archive
set file_id = (select new_file
from file_migration_temp
where original_file = file_id and attachment_archive.id is not null)
;
update attachment_preview
set file_id = (select new_file
from file_migration_temp
where original_file = file_id and attachment_preview.id is not null)
;
update classifier_model
set file_id = (select new_file
from file_migration_temp
where original_file = file_id and classifier_model.id is not null)
;
-- delete old filemeta and filechunk rows
delete from filemeta
where file_id in (select original_file from file_migration_temp);
delete from filemeta
where file_id in (select file_id from obsolete_files);
delete from filechunk
where file_id in (select file_id from obsolete_files);
-- update chunks
update filechunk
set file_id = (select new_file
from file_migration_temp
where original_file = file_id and filechunk.file_id is not null)
;
-- drop temp table
drop table file_migration_temp;
drop table obsolete_files;

150
modules/store/src/main/resources/db/migration/postgresql/V1.33.0__reorganize_file_ids.sql Normal file
View File

@ -0,0 +1,150 @@
drop table if exists file_migration_temp;
create table file_migration_temp (
id bigserial primary key,
original_file varchar(254) not null unique,
cid varchar(254) not null,
category varchar(254) not null,
new_file varchar(254) not null unique
);
-- Source files
insert into file_migration_temp (original_file, cid, category, new_file)
select
rs.file_id as original_file,
i.cid,
'attachmentsource' as category,
i.cid || '/attachmentsource/' || rs.file_id as new_file
from attachment_source rs
inner join attachment ra on rs.id = ra.attachid
inner join item i on ra.itemid = i.itemid
;
-- Archive files
insert into file_migration_temp (original_file, cid, category, new_file)
select distinct
rs.file_id as original_file,
i.cid,
'attachmentsource' as category,
i.cid || '/attachmentsource/' || rs.file_id as new_file
from attachment_archive rs
inner join attachment ra on rs.id = ra.attachid
inner join item i on ra.itemid = i.itemid
;
-- Converted files
insert into file_migration_temp (original_file, cid, category, new_file)
select
ra.filemetaid as original_file,
i.cid,
'attachmentconvert' as category,
i.cid || '/attachmentconvert/' || ra.filemetaid as new_file
from attachment_source rs
inner join attachment ra on rs.id = ra.attachid
inner join item i on ra.itemid = i.itemid
where rs.file_id <> ra.filemetaid
;
-- Preview image
insert into file_migration_temp (original_file, cid, category, new_file)
select
ap.file_id as original_file,
i.cid,
'previewimage' as category,
i.cid || '/previewimage/' || ap.file_id as new_file
from attachment_preview ap
inner join attachment ra on ra.attachid = ap.id
inner join item i on i.itemid = ra.itemid
order by id
;
-- classifier
insert into file_migration_temp (original_file, cid, category, new_file)
select
file_id as original_file,
cid,
'classifier' as category,
cid || '/classifier/' || file_id as new_file
from classifier_model
;
-- save obsolete/orphaned files
drop table if exists obsolete_files;
create table obsolete_files(
file_id varchar(254) not null,
mimetype varchar(254) not null,
length bigint not null,
checksum varchar(254) not null,
created timestamp not null
);
with
missing_ids as (
select file_id from filemeta
except
select original_file as file_id from file_migration_temp)
insert into obsolete_files (file_id, mimetype, length, checksum, created)
select file_id, mimetype, length, checksum, created from filemeta
where file_id in (select file_id from missing_ids)
;
-- duplicate each filemeta with the new id
insert into filemeta (file_id, mimetype, length, checksum, created)
select mm.new_file, fm.mimetype, fm.length, fm.checksum, fm.created
from file_migration_temp mm
inner join filemeta fm on fm.file_id = mm.original_file
;
-- update each reference to the new id
update attachment_source
set file_id = (select new_file
from file_migration_temp
where original_file = file_id and attachment_source.id is not null)
;
update attachment
set filemetaid = (select new_file
from file_migration_temp
where original_file = filemetaid and attachment.attachid is not null)
;
update attachment_archive
set file_id = (select new_file
from file_migration_temp
where original_file = file_id and attachment_archive.id is not null)
;
update attachment_preview
set file_id = (select new_file
from file_migration_temp
where original_file = file_id and attachment_preview.id is not null)
;
update classifier_model
set file_id = (select new_file
from file_migration_temp
where original_file = file_id and classifier_model.id is not null)
;
-- delete old filemeta and filechunk rows
delete from filemeta
where file_id in (select original_file from file_migration_temp);
delete from filemeta
where file_id in (select file_id from obsolete_files);
delete from filechunk
where file_id in (select file_id from obsolete_files);
-- update chunks
update filechunk
set file_id = (select new_file
from file_migration_temp
where original_file = file_id and filechunk.file_id is not null)
;
-- drop temp table
drop table file_migration_temp;
drop table obsolete_files;