567 lines
29 KiB
Python
567 lines
29 KiB
Python
#!/usr/bin/env python3
|
||
"""Повторяемая сортировка фотографий: анализ, ручная проверка и отдельный перенос."""
|
||
import argparse
|
||
import base64
|
||
import csv
|
||
import datetime as dt
|
||
import hashlib
|
||
import json
|
||
import os
|
||
from pathlib import Path
|
||
import shlex
|
||
import shutil
|
||
import sqlite3
|
||
import subprocess
|
||
import sys
|
||
import uuid
|
||
import warnings
|
||
from collections import Counter
|
||
from contextlib import contextmanager
|
||
|
||
import numpy as np
|
||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||
|
||
MONTHS = 'январь февраль март апрель май июнь июль август сентябрь октябрь ноябрь декабрь'.split()
|
||
FIELDS = ['id', 'action', 'date', 'source', 'reason', 'winner']
|
||
ACTIONS = {'sort', 'del', 'ras', 'skip', 'keep'}
|
||
IMAGE_EXT = {'.jpg', '.jpeg', '.png', '.tif', '.tiff', '.bmp', '.webp', '.gif', '.heic', '.heif', '.avif', '.dng', '.nef', '.cr2', '.cr3', '.arw', '.orf', '.rw2', '.psd'}
|
||
READABLE_EXT = {'.jpg', '.jpeg', '.png', '.tif', '.tiff', '.bmp', '.webp', '.gif'}
|
||
warnings.simplefilter('error', Image.DecompressionBombWarning)
|
||
|
||
|
||
def digest(path):
|
||
h = hashlib.sha256()
|
||
with Path(path).open('rb') as f:
|
||
for chunk in iter(lambda: f.read(1024 * 1024), b''):
|
||
h.update(chunk)
|
||
return h.hexdigest()
|
||
|
||
|
||
def date_value(value):
|
||
if isinstance(value, bytes):
|
||
value = value.decode('ascii', 'ignore')
|
||
if not isinstance(value, str):
|
||
return ''
|
||
for fmt in ('%Y:%m:%d %H:%M:%S', '%Y-%m-%d %H:%M:%S', '%Y-%m-%dT%H:%M:%S'):
|
||
try:
|
||
date = dt.datetime.strptime(value[:19], fmt)
|
||
if 1800 <= date.year <= dt.datetime.now().year:
|
||
return date.isoformat()
|
||
except ValueError:
|
||
pass
|
||
return ''
|
||
|
||
|
||
def resolve_dates(row, values):
|
||
row['date_candidates'] = {k: date_value(v) for k, v in values.items() if date_value(v)}
|
||
dates = set(row['date_candidates'].values())
|
||
row['date'] = next(iter(dates)) if len(dates) == 1 else ''
|
||
row['reason'] = ('дата создания контента из метаданных' if len(dates) == 1 else
|
||
'противоречивые даты контента' if dates else 'нет встроенной даты контента')
|
||
|
||
|
||
def inspect(path, cache):
|
||
before = path.stat()
|
||
sha = digest(path)
|
||
if sha in cache:
|
||
result = dict(cache[sha])
|
||
else:
|
||
result = dict(sha=sha, photo=False, reason='не фото или неподдерживаемый формат', date='', pixels=0, width=0, height=0)
|
||
if path.suffix.lower() in IMAGE_EXT:
|
||
try:
|
||
with Image.open(path) as im:
|
||
if getattr(im, 'n_frames', 1) > 1:
|
||
result['reason'] = 'многостраничное изображение: ручная проверка'
|
||
else:
|
||
# Ошибочные метаданные сами по себе не делают читаемую фотографию повреждённой.
|
||
try:
|
||
exif = im.getexif()
|
||
nested = exif.get_ifd(34665) if 34665 in exif else {}
|
||
values = {'EXIF:DateTimeOriginal': nested.get(36867) or exif.get(36867),
|
||
'EXIF:CreateDate': nested.get(36868) or exif.get(36868)}
|
||
resolve_dates(result, values)
|
||
except (ValueError, SyntaxError, OSError):
|
||
pass
|
||
im.load()
|
||
try:
|
||
upright = ImageOps.exif_transpose(im)
|
||
except (ValueError, SyntaxError, OSError):
|
||
upright = im.copy()
|
||
alpha = 'A' in upright.getbands() or 'transparency' in upright.info
|
||
pixels = upright.convert('RGBA' if alpha else 'RGB')
|
||
rgb = upright.convert('RGB')
|
||
small = np.asarray(rgb.resize((9, 8), Image.Resampling.LANCZOS, reducing_gap=3).convert('L'))
|
||
bits = (small[:, 1:] > small[:, :-1]).flat
|
||
result.update(photo=True,
|
||
width=im.width, height=im.height, pixels=im.width * im.height,
|
||
pixelsha=hashlib.sha256(str((pixels.size, pixels.mode)).encode() + pixels.tobytes()).hexdigest(),
|
||
dh=sum(int(b) << i for i, b in enumerate(bits)), alpha=alpha,
|
||
thumb=rgb.resize((64, 64), Image.Resampling.LANCZOS, reducing_gap=3).tobytes())
|
||
result['width'], result['height'] = upright.size
|
||
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
|
||
result['reason'] = 'превышен лимит размера: ручная проверка'
|
||
except (UnidentifiedImageError, OSError, SyntaxError, ValueError) as error:
|
||
if isinstance(error, OSError) and error.errno is not None:
|
||
raise
|
||
result['broken'] = path.suffix.lower() in READABLE_EXT
|
||
result['reason'] = 'ошибка декодирования: ' + str(error)
|
||
cache[sha] = dict(result)
|
||
after = path.stat()
|
||
if (before.st_size, before.st_mtime_ns) != (after.st_size, after.st_mtime_ns):
|
||
raise RuntimeError('Файл изменился при чтении: ' + str(path))
|
||
result.update(source=str(path), size=after.st_size, mtime=after.st_mtime_ns)
|
||
return result
|
||
|
||
|
||
def walk(root, excluded=()):
|
||
def fail(error):
|
||
raise error
|
||
for folder, dirs, files in os.walk(root, followlinks=False, onerror=fail):
|
||
dirs[:] = sorted(d for d in dirs if d not in excluded and not (Path(folder) / d).is_symlink())
|
||
for name in sorted(files):
|
||
path = Path(folder) / name
|
||
if path.is_file() and not path.is_symlink():
|
||
yield path
|
||
|
||
|
||
def fingerprint(path):
|
||
st = path.stat()
|
||
return json.dumps([3, Image.__version__, Image.MAX_IMAGE_PIXELS, st.st_dev, st.st_ino, st.st_size, st.st_mtime_ns, st.st_ctime_ns])
|
||
|
||
|
||
def database(path):
|
||
path = Path(path).expanduser().resolve()
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
db = sqlite3.connect(path, timeout=60)
|
||
db.execute('CREATE TABLE IF NOT EXISTS files (path TEXT PRIMARY KEY, signature TEXT, data TEXT)')
|
||
db.execute('CREATE TABLE IF NOT EXISTS catalog (archive TEXT, path TEXT, sha TEXT, group_id TEXT, state TEXT, data TEXT, PRIMARY KEY(archive,path))')
|
||
db.execute('CREATE TABLE IF NOT EXISTS history (id INTEGER PRIMARY KEY, run TEXT, source TEXT, target TEXT, sha TEXT, created TEXT)')
|
||
db.commit()
|
||
return db
|
||
|
||
|
||
def encoded_row(row):
|
||
value = dict(row)
|
||
if isinstance(value.get('thumb'), bytes):
|
||
value['thumb'] = base64.b64encode(value['thumb']).decode('ascii')
|
||
return json.dumps(value, ensure_ascii=False)
|
||
|
||
|
||
def remember(db, archive, row, state):
|
||
db.execute('INSERT OR REPLACE INTO catalog VALUES (?, ?, ?, ?, ?, ?)',
|
||
(archive, row['source'], row['sha'], row['group_id'], state, encoded_row(row)))
|
||
|
||
|
||
def cached_inspect(path, memory, db):
|
||
key = fingerprint(path)
|
||
cached = db.execute('SELECT signature, data FROM files WHERE path=?', (str(path),)).fetchone()
|
||
if cached and cached[0] == key:
|
||
row = json.loads(cached[1])
|
||
if 'thumb' in row:
|
||
row['thumb'] = base64.b64decode(row['thumb'])
|
||
return row
|
||
row = inspect(path, memory)
|
||
encoded = dict(row)
|
||
if 'thumb' in encoded:
|
||
encoded['thumb'] = base64.b64encode(encoded['thumb']).decode('ascii')
|
||
if fingerprint(path) != key:
|
||
raise RuntimeError('Файл изменился при анализе: ' + str(path))
|
||
db.execute('INSERT OR REPLACE INTO files VALUES (?, ?, ?)', (str(path), key, json.dumps(encoded, ensure_ascii=False)))
|
||
return row
|
||
|
||
|
||
def extra_dates(rows, executable):
|
||
groups = {}
|
||
for row in rows:
|
||
if row['photo'] and not row.get('metadata_checked'):
|
||
groups.setdefault(row['sha'], []).append(row)
|
||
reps = [g[0] for g in groups.values()]
|
||
for start in range(0, len(reps), 100):
|
||
batch = reps[start:start+100]
|
||
command = [executable, '-json', '-G1', '-EXIF:DateTimeOriginal', '-EXIF:CreateDate',
|
||
'-XMP-exif:DateTimeOriginal', '-XMP-xmp:CreateDate', '-XMP-photoshop:DateCreated',
|
||
'-IPTC:DateCreated', '-IPTC:TimeCreated']
|
||
command += [r['source'] for r in batch]
|
||
output = subprocess.run(command, capture_output=True, text=True)
|
||
if output.returncode not in (0, 1):
|
||
raise RuntimeError(output.stderr)
|
||
values = {r['SourceFile']: r for r in json.loads(output.stdout)}
|
||
for r in batch:
|
||
v = values.get(r['source'], {})
|
||
dates = dict(r.get('date_candidates', {}))
|
||
dates.update({k: value for k, value in v.items() if k.endswith((':DateTimeOriginal', ':CreateDate'))
|
||
or k == 'XMP-photoshop:DateCreated'})
|
||
if v.get('IPTC:DateCreated') and v.get('IPTC:TimeCreated'):
|
||
dates['IPTC:ContentCreated'] = v['IPTC:DateCreated'] + ' ' + v['IPTC:TimeCreated']
|
||
for item in groups[r['sha']]:
|
||
resolve_dates(item, dates)
|
||
item['metadata_checked'] = True
|
||
print(f'Метаданные: {min(start+100, len(reps))}/{len(reps)}', flush=True)
|
||
|
||
|
||
def classify(rows, visual):
|
||
exact, decoded, buckets = {}, {}, {}
|
||
parents = {}
|
||
def group(key):
|
||
while key in parents:
|
||
key = parents[key]
|
||
return key
|
||
for r in sorted(rows, key=lambda r: (-r['pixels'], -r['size'], not r['existing'], r['source'])):
|
||
r['action'] = 'keep' if r['existing'] else ('del' if r.get('broken') else 'ras')
|
||
if not r['photo']:
|
||
continue
|
||
match = exact.get(r['sha']) or decoded.get(r['pixelsha'])
|
||
reason = 'одинаковый файл или декодированные пиксели'
|
||
if not match and visual and not r['alpha']:
|
||
candidates = {}
|
||
for b in range(4):
|
||
for other in buckets.get((b, (r['dh'] >> (b*16)) & 65535), []):
|
||
candidates[other['source']] = other
|
||
for other in candidates.values():
|
||
if other['alpha'] or (r['dh'] ^ other['dh']).bit_count() > 2:
|
||
continue
|
||
if abs(r['width']/r['height'] - other['width']/other['height']) > .002:
|
||
continue
|
||
a = np.frombuffer(r['thumb'], dtype=np.uint8).astype(np.float32)
|
||
b = np.frombuffer(other['thumb'], dtype=np.uint8).astype(np.float32)
|
||
if min(a.std(), b.std()) >= 12 and np.sqrt(np.mean((a-b)**2)) <= 1.5 and np.quantile(np.abs(a-b), .99) <= 5:
|
||
match, reason = other, 'визуальный дубль: проверьте пару'
|
||
break
|
||
if match:
|
||
r.update(action='del', winner=match['source'], reason=reason)
|
||
old, new = group(r['group_id']), group(match['group_id'])
|
||
if old != new:
|
||
parents[old] = new
|
||
r['group_id'] = match['group_id']
|
||
else:
|
||
r['action'] = 'keep' if r['existing'] and r['date'] else ('sort' if r['date'] else 'ras')
|
||
exact[r['sha']] = decoded[r['pixelsha']] = r
|
||
for b in range(4):
|
||
buckets.setdefault((b, (r['dh'] >> (b*16)) & 65535), []).append(r)
|
||
for r in rows:
|
||
r['group_id'] = group(r['group_id'])
|
||
return {key: group(key) for key in parents}
|
||
|
||
|
||
def write_review(run, rows):
|
||
temporary = run / 'review.tmp'
|
||
with temporary.open('w', newline='', encoding='utf-8-sig') as f:
|
||
writer = csv.DictWriter(f, fieldnames=FIELDS)
|
||
writer.writeheader()
|
||
for r in rows:
|
||
writer.writerow({key: r.get(key, '') for key in FIELDS})
|
||
temporary.replace(run / 'review.csv')
|
||
|
||
|
||
def load_review(run):
|
||
plan = json.loads((run / 'plan.json').read_text())
|
||
if plan.get('version') != 2:
|
||
raise ValueError('План создан по старым правилам даты. Выполните новый анализ; старые отчёты сохранены.')
|
||
with (run / 'review.csv').open(encoding='utf-8-sig', newline='') as f:
|
||
edited = list(csv.DictReader(f))
|
||
if len(edited) != len(plan['rows']) or len({r['id'] for r in edited}) != len(edited):
|
||
raise ValueError('В review.csv изменён состав строк')
|
||
edits = {r['id']: r for r in edited}
|
||
for row in plan['rows']:
|
||
edit = edits[row['id']]
|
||
if edit['source'] != row['source'] or edit['action'] not in ACTIONS:
|
||
raise ValueError('Недопустимый путь или действие в CSV')
|
||
row['action'] = edit['action']
|
||
original_date = row['date']
|
||
row['date'] = date_value(edit['date']) if edit['date'] else ''
|
||
if row['date'] and row['date'] != original_date:
|
||
row['reason'] = 'дата назначена вручную'
|
||
row['manual_date'] = row['date']
|
||
if edit['date'] and not row['date']:
|
||
raise ValueError('Неверная дата: ' + edit['date'])
|
||
if row['action'] == 'sort' and (not row['date'] or not row['photo']):
|
||
raise ValueError('Для sort нужны читаемое фото и полная дата: ' + row['source'])
|
||
return plan
|
||
|
||
|
||
def analyze(args):
|
||
source, dest = Path(args.source).expanduser().resolve(), Path(args.dest).expanduser().resolve()
|
||
if not source.is_dir() or not dest.is_dir():
|
||
raise ValueError('Исходная папка и архив должны существовать')
|
||
run = Path(args.run).expanduser().resolve() if args.run else Path.home()/'.photo-sort'/'runs'/(dt.datetime.now().strftime('%Y%m%d-%H%M%S')+'-'+uuid.uuid4().hex[:6])
|
||
if run.is_relative_to(source) or run.is_relative_to(dest):
|
||
raise ValueError('Папка отчёта должна быть вне исходной папки и архива')
|
||
run.mkdir(parents=True, exist_ok=False)
|
||
if args.select:
|
||
if any((source / p).is_symlink() for p in args.select):
|
||
raise ValueError('Символические ссылки не обрабатываются')
|
||
paths = sorted({(source / p).resolve() for p in args.select})
|
||
if any(not p.is_relative_to(source) or not p.is_file() or p.is_symlink() for p in paths):
|
||
raise ValueError('Выбранные файлы должны находиться в исходной папке')
|
||
else:
|
||
paths = list(walk(source, ('_DEL', '_RAS')))
|
||
selected = set(paths)
|
||
if not args.no_compare:
|
||
paths += [p for p in walk(dest, ('_DEL', 'BACKUP')) if p not in selected and not p.is_relative_to(source)]
|
||
cache, rows = {}, []
|
||
Image.MAX_IMAGE_PIXELS = args.max_megapixels * 1_000_000
|
||
cache_path = Path(args.database).expanduser().resolve()
|
||
with database(cache_path) as db:
|
||
known = {p: (sha, gid, json.loads(data)) for p, sha, gid, data in db.execute('SELECT path, sha, group_id, data FROM catalog WHERE archive=?', (str(dest),))}
|
||
for i, path in enumerate(paths, 1):
|
||
row = cached_inspect(path, cache, db)
|
||
old = known.get(str(path))
|
||
row.update(existing=path not in selected, id=str(i), winner='',
|
||
group_id=old[1] if old and old[0] == row['sha'] else row['sha'])
|
||
rows.append(row)
|
||
if i % 50 == 0 or i == len(paths):
|
||
db.commit()
|
||
print(f'Проверено: {i}/{len(paths)}', flush=True)
|
||
exiftool = args.exiftool or shutil.which('exiftool')
|
||
bundled = Path(__file__).parent / 'vendor' / 'exiftool' / 'exiftool'
|
||
if not exiftool and bundled.exists():
|
||
exiftool = str(bundled)
|
||
if exiftool:
|
||
extra_dates(rows, exiftool)
|
||
else:
|
||
print('ExifTool не найден: доступны только основные метаданные Pillow.')
|
||
# Сохраняем метаданные до ручных решений: они не должны попадать в кэш декодирования.
|
||
with database(cache_path) as db:
|
||
for row in rows:
|
||
db.execute('UPDATE files SET data=? WHERE path=?', (encoded_row(row), row['source']))
|
||
for row in rows:
|
||
prior = known.get(row['source'])
|
||
if prior and prior[0] == row['sha'] and prior[2].get('manual_date'):
|
||
row.update(date=prior[2]['manual_date'], manual_date=prior[2]['manual_date'], reason='ранее назначенная вручную дата этого файла')
|
||
if args.date:
|
||
date = date_value(args.date)
|
||
if not date:
|
||
raise ValueError('Дата должна быть YYYY-MM-DD HH:MM:SS')
|
||
for row in rows:
|
||
if not row['existing'] and row['photo']:
|
||
row.update(date=date, manual_date=date, reason='дата назначена вручную')
|
||
merges = classify(rows, args.visual)
|
||
with database(cache_path) as db:
|
||
for old, new in merges.items():
|
||
db.execute('UPDATE catalog SET group_id=? WHERE archive=? AND group_id=?', (new, str(dest), old))
|
||
for (path,) in db.execute('SELECT path FROM catalog WHERE archive=? AND state != ?', (str(dest), 'missing')).fetchall():
|
||
if not Path(path).is_file():
|
||
db.execute('UPDATE catalog SET state=? WHERE archive=? AND path=?', ('missing', str(dest), path))
|
||
for row in rows:
|
||
remember(db, str(dest), row, 'archive' if row['existing'] else 'incoming')
|
||
for row in rows:
|
||
row.pop('thumb', None)
|
||
(run/'plan.json').write_text(json.dumps(dict(version=2, database=str(cache_path), source=str(source), dest=str(dest), rows=rows), ensure_ascii=False, indent=2))
|
||
write_review(run, rows)
|
||
print('План:', run)
|
||
print(dict(Counter(r['action'] for r in rows)))
|
||
print('Файлы не перемещены. Команды: review, show, apply.')
|
||
return run
|
||
|
||
|
||
def targets(plan):
|
||
dest = Path(plan['dest'])
|
||
reserved, counters, operations = set(), {}, []
|
||
released = {r['source'] for r in plan['rows'] if r['action'] == 'del' and Path(r['source']).parent != dest/'_DEL'}
|
||
for r in plan['rows']:
|
||
if r['action'] in ('keep', 'skip'):
|
||
continue
|
||
src = Path(r['source'])
|
||
if r['action'] == 'sort':
|
||
date = dt.datetime.fromisoformat(r['date'])
|
||
target = dest/str(date.year)/MONTHS[date.month-1]/(date.strftime('%d%m%Y%H%M%S') + src.suffix.lower())
|
||
else:
|
||
target = dest/('_DEL' if r['action']=='del' else '_RAS')/src.name
|
||
if target == src:
|
||
continue
|
||
base, n = target, counters.get(str(target), 1)
|
||
while str(target).casefold() in reserved or (target.exists() and str(target) not in released) or target.is_symlink():
|
||
target = base.with_name(base.stem + ('_dub' if n==1 else f'_dub{n}') + base.suffix)
|
||
n += 1
|
||
counters[str(base)] = n
|
||
reserved.add(str(target).casefold())
|
||
if not target.resolve().is_relative_to(dest):
|
||
raise ValueError('Путь назначения выходит за пределы архива')
|
||
operations.append(dict(r, target=str(target)))
|
||
return sorted(operations, key=lambda r: r['action'] != 'del')
|
||
|
||
|
||
def show(run):
|
||
plan = load_review(run)
|
||
ops = targets(plan)
|
||
print(dict(Counter(r['action'] for r in plan['rows'])))
|
||
print(f'Перемещений: {len(ops)}; из существующего архива: {sum(r["existing"] for r in ops)}')
|
||
for r in ops[:20]:
|
||
print(r['source'], '->', r['target'])
|
||
return ops
|
||
|
||
|
||
@contextmanager
|
||
def locked(run):
|
||
import fcntl
|
||
with (run/'operation.lock').open('a') as f:
|
||
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||
yield
|
||
|
||
|
||
def emit(log, **record):
|
||
log.write(json.dumps(record, ensure_ascii=False)+'\n')
|
||
log.flush()
|
||
os.fsync(log.fileno())
|
||
|
||
|
||
def apply(run, yes=False):
|
||
plan = load_review(run)
|
||
lockdir = Path(plan['database']).parent/'locks'/hashlib.sha256(plan['dest'].encode()).hexdigest()
|
||
lockdir.mkdir(parents=True, exist_ok=True)
|
||
with locked(lockdir):
|
||
return apply_locked(run, yes)
|
||
|
||
|
||
def record_move(plan, run, row, src, dst):
|
||
with database(plan['database']) as db:
|
||
cached = db.execute('SELECT data FROM files WHERE path=?', (str(src),)).fetchone()
|
||
if cached:
|
||
value = json.loads(cached[0])
|
||
value.update(source=str(dst), mtime=dst.stat().st_mtime_ns)
|
||
db.execute('INSERT OR REPLACE INTO files VALUES (?, ?, ?)', (str(dst), fingerprint(dst), json.dumps(value, ensure_ascii=False)))
|
||
db.execute('UPDATE catalog SET state=? WHERE archive=? AND path=?', ('moved', plan['dest'], str(src)))
|
||
remember(db, plan['dest'], dict(row, source=str(dst)), 'del' if row['action']=='del' else 'ras' if row['action']=='ras' else 'archive')
|
||
db.execute('INSERT INTO history(run,source,target,sha,created) VALUES (?,?,?,?,?)',
|
||
(str(run), str(src), str(dst), row['sha'], dt.datetime.now().isoformat()))
|
||
|
||
|
||
def apply_locked(run, yes=False):
|
||
with locked(run):
|
||
journal = run/'moves.jsonl'
|
||
if journal.exists():
|
||
raise RuntimeError('Этот запуск уже применялся. Для новой обработки создайте новый план. При сбое проверьте moves.jsonl; исходники незавершённых копий сохранены.')
|
||
plan = load_review(run)
|
||
ops = show(run)
|
||
if not yes and input('Для выполнения введите ПЕРЕНЕСТИ: ').strip() != 'ПЕРЕНЕСТИ':
|
||
print('Отменено')
|
||
return
|
||
# Проверяем и оставляемые оригиналы, чтобы не применить устаревшее решение о дубле.
|
||
for r in plan['rows']:
|
||
p = Path(r['source'])
|
||
if p.is_symlink() or not p.is_file() or p.stat().st_size != r['size'] or digest(p) != r['sha']:
|
||
raise RuntimeError('Файл изменён или отсутствует; нужен новый анализ: '+str(p))
|
||
with journal.open('x') as log:
|
||
for i, r in enumerate(ops, 1):
|
||
src, dst = Path(r['source']), Path(r['target'])
|
||
if not dst.resolve().is_relative_to(Path(plan['dest'])):
|
||
raise RuntimeError('Изменён путь назначения')
|
||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||
emit(log, state='intent', source=str(src), target=str(dst), sha=r['sha'])
|
||
# Эксклюзивное создание работает и между разными дисками, включая exFAT.
|
||
# Исходник удаляется только после проверки копии.
|
||
with dst.open('xb') as out, src.open('rb') as inp:
|
||
shutil.copyfileobj(inp, out, 1024*1024)
|
||
out.flush()
|
||
os.fsync(out.fileno())
|
||
if digest(dst) != r['sha'] or digest(src) != r['sha']:
|
||
raise RuntimeError('Проверка копии не прошла; исходник сохранён: '+str(src))
|
||
os.utime(dst, ns=(src.stat().st_atime_ns, src.stat().st_mtime_ns))
|
||
emit(log, state='verified', source=str(src), target=str(dst), sha=r['sha'])
|
||
src.unlink()
|
||
record_move(plan, run, r, src, dst)
|
||
emit(log, state='done', source=str(src), target=str(dst), sha=r['sha'])
|
||
if i % 50 == 0 or i == len(ops):
|
||
print(f'Перемещено: {i}/{len(ops)}', flush=True)
|
||
print('Готово. Журнал:', journal)
|
||
|
||
|
||
def review(run, only):
|
||
with locked(run):
|
||
if (run/'moves.jsonl').exists():
|
||
raise ValueError('Этот план уже применялся')
|
||
plan = load_review(run)
|
||
rows = plan['rows']
|
||
for r in rows:
|
||
if only == 'unresolved' and r['action'] != 'ras':
|
||
continue
|
||
if only == 'duplicates' and not r['winner']:
|
||
continue
|
||
while True:
|
||
print('\n', r['id'], r['source'], '\n', r['reason'], '\nДата:', r['date'] or 'нет', 'Действие:', r['action'])
|
||
answer = input('[o] открыть, [w] оригинал, [t] дата, [d] _DEL, [r] _RAS, [s] пропустить, Enter далее, [q] выход: ').strip().lower()
|
||
if answer in ('o', 'w'):
|
||
path = r['source'] if answer == 'o' else r['winner']
|
||
if path:
|
||
subprocess.run(['open' if sys.platform=='darwin' else 'xdg-open', path], check=False)
|
||
elif answer == 't':
|
||
value = date_value(input('Дата YYYY-MM-DD HH:MM:SS: '))
|
||
if value and r['photo']:
|
||
r.update(date=value, action='sort')
|
||
write_review(run, rows)
|
||
else:
|
||
print('Нужны полная дата и читаемое фото.')
|
||
elif answer in ('d', 'r', 's'):
|
||
r['action'] = {'d':'del','r':'ras','s':'skip'}[answer]
|
||
write_review(run, rows)
|
||
break
|
||
elif answer == 'q':
|
||
return
|
||
elif not answer:
|
||
break
|
||
|
||
|
||
def main(argv=None):
|
||
parser = argparse.ArgumentParser(description='Сортировка фото: анализ -> ручная проверка -> перенос. Без удаления содержимого.')
|
||
commands = parser.add_subparsers(dest='command', required=True)
|
||
p = commands.add_parser('analyze', help='Создать новый план, не перемещая файлы')
|
||
p.add_argument('--source', required=True); p.add_argument('--dest', required=True)
|
||
p.add_argument('--select', nargs='+', help='Только выбранные файлы внутри source')
|
||
p.add_argument('--date', help='Назначить выбранным фото YYYY-MM-DD HH:MM:SS')
|
||
p.add_argument('--run', help='Новая папка отчёта вне source и dest')
|
||
p.add_argument('--visual', action='store_true', help='Также искать визуальные дубли; затем проверьте review --only duplicates')
|
||
p.add_argument('--no-compare', action='store_true', help='Не сравнивать с существующим архивом')
|
||
p.add_argument('--exiftool'); p.add_argument('--max-megapixels', type=int, default=100)
|
||
p.add_argument('--database', default=str(Path.home()/'.photo-sort'/'cache.sqlite'), help='Постоянная база, не удаляется между запусками')
|
||
for name in ('review', 'show', 'apply'):
|
||
p = commands.add_parser(name)
|
||
p.add_argument('run', type=lambda s: Path(s).expanduser().resolve())
|
||
if name=='review': p.add_argument('--only', choices=['unresolved','duplicates','all'], default='unresolved')
|
||
if name=='apply': p.add_argument('--yes', action='store_true', help='Подтвердить перенос без вопроса')
|
||
args = parser.parse_args(argv)
|
||
if args.command=='analyze': return analyze(args)
|
||
if args.command=='review': return review(args.run, args.only)
|
||
if args.command=='show': return show(args.run)
|
||
if args.command=='apply': return apply(args.run, args.yes)
|
||
|
||
|
||
def menu():
|
||
while True:
|
||
print('\nФото: 1 анализ папки, 2 ручная проверка, 3 показать план, 4 выполнить перенос, 0 выход')
|
||
choice = input('> ').strip()
|
||
if choice=='0': return
|
||
if choice=='1':
|
||
source = input('Исходная папка (можно _RAS): ').strip().strip('"\'')
|
||
dest = input('Архив FOTO: ').strip().strip('"\'')
|
||
date = input('Дата для всех выбранных фото (Enter = метаданные): ').strip()
|
||
selected = shlex.split(input('Имена выбранных файлов в кавычках (Enter = вся папка): '))
|
||
argv = ['analyze','--source',source,'--dest',dest]
|
||
if date: argv += ['--date',date]
|
||
if selected: argv += ['--select',*selected]
|
||
if input('Искать визуальные дубли? [д/Н]: ').lower()=='д': argv += ['--visual']
|
||
elif choice in ('2','3','4'):
|
||
run = input('Папка отчёта: ').strip().strip('"\'')
|
||
argv = [{'2':'review','3':'show','4':'apply'}[choice],run]
|
||
if choice=='2':
|
||
group = input('1 без даты/неразобранные, 2 дубли, 3 весь план [1]: ').strip()
|
||
argv += ['--only', {'2':'duplicates','3':'all'}.get(group, 'unresolved')]
|
||
else:
|
||
continue
|
||
try:
|
||
main(argv)
|
||
except (OSError, ValueError, RuntimeError, KeyError) as error:
|
||
print('Ошибка:', error)
|
||
|
||
|
||
if __name__ == '__main__':
|
||
try:
|
||
menu() if len(sys.argv)==1 else main()
|
||
except KeyboardInterrupt:
|
||
print('\nОстановлено. План и журнал сохранены, если были созданы.')
|
||
sys.exit(130)
|
||
except EOFError:
|
||
print('\nВыход')
|
||
except (OSError, ValueError, RuntimeError, KeyError) as error:
|
||
print('Ошибка:', error, file=sys.stderr)
|
||
sys.exit(1)
|