import re import os from pathlib import Path import time from datetime import datetime import subprocess import pickle from zoneinfo import ZoneInfo if __name__ == '__main__': import django import sys sys.path.insert(0, str(Path(__file__).parent.parent)) os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mysrc.settings') django.setup() from tikz.models import Tikz LATEXTEMPLATE = r""" \documentclass[11pt]{article} \usepackage[fourier]{PNS} \usepackage{tikz} \usetikzlibrary{arrows.meta,calc,decorations.pathmorphing} \tikzset{% every picture/.style={>=Stealth}, vel/.style={->,line width=2pt,color=DarkBlue}, vector/.style={->,line width=2pt, color=SlateBlue}, force/.style={line width=1.5pt,color=blue,->}, coord/.style={color=green!40!black,->}, accel/.style={->,line width=2.5pt,color=gray}, photon/.style={line width=1.5pt,color=DarkRed,decorate,decoration={snake,post length=0.1in}}, spring/.style={decorate,decoration={coil,aspect=0.3,segment length=2mm,amplitude=2mm}}, traj/.style={dashed, color=gray, line width=1pt}, component/.style={->,dashed,line width=1pt, color=SlateGray} } \definecolor{trace01}{rgb}{0.5,0,0} \definecolor{trace02}{rgb}{0,0,0.5} \definecolor{trace03}{rgb}{0,0.5.0} \definecolor{trace04}{rgb}{0.5,0.5,0} \definecolor{trace05}{rgb}{0.4,0.3,0} \usepackage{pgfplots} \pgfplotsset{compat=1.18,width=3in} DEFS \begin{document} \thispagestyle{empty} SOURCE \end{document} """ class Loader: HOME = Path(__file__).parent.parent DBFILE = HOME / 'files.pickle' DATABASE = None def __init__(self, path:str): """ Make sure that path is a complete path, then turn it into a hash code, look for the database of this base path and load it, if found. The database holds filenames and modtimes. Any file in the database that has a modtime that matches the one on disk is presumed to have been properly scanned already. In walking the tree, this model would need to """ self.path = Path(path).absolute() self.tmp = self.HOME / 'tmp' logs = self.tmp / os.path.split(path)[1] self.errors = str(logs / 'err.log') self.success = str(logs / 'success.log') for v in (self.errors, self.success): open(v, 'w').write(f"Start at {time.asctime()}\n\n") zone = ZoneInfo('America/Los_Angeles') self._load_db() db = self.DATABASE for dirpath, dirnames, filenames in os.walk(self.path): for name in filenames: if name.startswith('.') or not name.endswith('.tex'): continue # Check for presence in the database p = Path(dirpath) / name key = str(p.absolute()) try: mtime = p.stat().st_mtime except: continue moddt = datetime.fromtimestamp(mtime, tz=zone) if key in db: # check the modtime if db[key] == mtime: continue else: db[key] = mtime self.process(p, moddt) self._save_db() def process(self, path, moddt): qs = Tikz.objects.filter(folder=str(path.parent), filename=path.name) repeat = qs.count() > 0 if repeat: dts = [] for tz in qs.all(): dt = moddt - tz.modified dts.append(abs(dt.total_seconds())) if max(dts) < 10: print(f"Nothing new in {path}") return source = "" if path.stat().st_size == 0: print(f"{path} is empty") return for encoding in ('utf-8', 'iso-8859-1'): try: source = open(path, 'r', encoding=encoding).read() break except UnicodeDecodeError: pass if len(source) == 0: raise Exception(f"Unable to decode file {path}") defs = self.getdefs(source) matches = re.finditer(r'\\begin{tikzpicture}.*?\\end{tikzpicture}', source, re.DOTALL+re.MULTILINE) savedir = os.getcwd() os.chdir(self.tmp) try: for match in matches: open('src.tex', 'w').write(match.group(0)) res = subprocess.run( ['latexindent', 'src.tex', '-o', 'fixed.tex'], capture_output=True) formatted = open('fixed.tex', 'r').read().replace('\t', ' ') # Do we already have this one? qs = Tikz.objects.filter(source=formatted) if qs.count() > 0: print(f"Found matching item width id {qs[0].id}") continue # Generate image open('img.tex', 'w').write( LATEXTEMPLATE.replace('SOURCE', formatted).replace('DEFS', defs) ) res = subprocess.run(['pdflatex', '-interaction=nonstopmode', 'img'], capture_output=True) log = res.stdout.decode() pdf = "Output written on img.pdf" in log errs = re.findall(r'^!.*$|Latex Error:.*$', log, re.M) with open(self.success if pdf else self.errors, 'a') as f: print('+' * 60, file=f) print(path, file=f) print("\n".join(errs), file=f) print('-' * 60, file=f) print("\n", file=f) if not pdf: raise Exception("Compile failed") subprocess.run(['pdfcrop', 'img', 'cropped.pdf']) subprocess.run(['convert', '-density', '150', 'cropped.pdf', 'cropped.png']) span = match.span() line_number = 1 + source[: span[0]].count('\n') tikz = Tikz.objects.create( folder=str(path.parent), filename=str(path.name), modified=moddt, line=line_number, source=formatted, thumbnail=open('cropped.png', 'rb').read(), domain=self.get_domain(path.parent) ) print(tikz) except: pass os.chdir(savedir) def get_domain(self, path): path = str(path) if not path.endswith('/'): path += '/' domains = [r'/Old Courses/', r'/Courses/', r'/Committees/', r'/Personal/'] for d in domains: m = re.search(d + '(.*?)/', path) if m: return m.group(1) domains = ['Chair', 'References'] for d in domains: m = re.search(r'/(' + d + ')/', path) if m: return m.group(1) if 'smbook' in path: return 'Book' print(f"Don't know about {path}") return "" def getdefs(self, t: str): """ Search for relevant definitions """ defs = [] try: for m in re.finditer( r'\\(renewcommand|newcommand|providecommand)(.*?)(\{.*?\})', t ): stack = [] net = m.group(0).count('{') - m.group(0).count('}') if net: for n in range(net): stack.append('{') txt = t[m.end() :] for n, char in enumerate(txt): if char == '{': stack.append(char) elif char == '}': stack.pop() if len(stack) == 0: defs.append( '\\' + m.group(1) + m.group(2) + m.group(3) + txt[0 : n + 1] ) break for m in re.finditer(r'\\tikzset\{', t): stack = [] txt = t[m.start() :] for n, char in enumerate(txt): if char == '{': stack.append(char) elif char == '}': stack.pop() if len(stack) == 0: defs.append(txt[: n + 1]) break except: pass sdefs = "\n".join(defs) return sdefs.replace("stealth'", 'Stealth') @staticmethod def _load_db(): if Loader.DATABASE is None: try: with open(Loader.DBFILE, 'rb') as f: Loader.DATABASE = pickle.load(f) except: Loader.DATABASE = dict() @staticmethod def _save_db(): with open(Loader.DBFILE, 'wb') as f: pickle.dump(Loader.DATABASE, f) if __name__ == '__main__': l = Loader('/Users/saeta/Documents/Courses') print('done')