Blame indexer.py

Ivan Mahonin ae689c
#!/usr/bin/python3
Ivan Mahonin ae689c
4cc0a2
# to forece reindex pass a single argument:
4cc0a2
#   force
4cc0a2
4cc0a2
# to 'hide' files
4cc0a2
# NB: file still will be accessible for ALL who know a mangled url
4cc0a2
#   place file into '.hide' subdirectory
4cc0a2
4cc0a2
# to set user thumbs:
4cc0a2
#   path.thumb.png      - user thumb for 'path'
4cc0a2
#   path/.thumb.png     - user thumb for 'path'
4cc0a2
#   path/.thumb..png    - default thumb for unknown extension
4cc0a2
#   path/.thumb...png   - default thumb for subdirectories
4cc0a2
#   path/.thumb.ext.png - default thumb for 'ext'
4cc0a2
#   path/.thumb/.png    - default thumb for unknown extension
4cc0a2
#   path/.thumb/..png   - default thumb for subdirectories
4cc0a2
#   path/.thumb/ext.png - default thumb for 'ext'
Ivan Mahonin ae689c
Ivan Mahonin ae689c
Ivan Mahonin ae689c
import os
Ivan Mahonin ae689c
import sys
Ivan Mahonin ae689c
import time
Ivan Mahonin cad502
import base64
Ivan Mahonin ae689c
import hashlib
root 8883a2
import urllib.parse
Ivan Mahonin ae689c
Ivan Mahonin ae689c
import config
Ivan Mahonin ae689c
import thumb
Ivan Mahonin ae689c
import template
Ivan Mahonin ae689c
Ivan Mahonin ae689c
Ivan Mahonin ae689c
thumbext = [ 'png', 'svg', 'jpg' ]
Ivan Mahonin ae689c
skipname = [ 'index.html', 'indexd.html' ]
Ivan Mahonin ae689c
Ivan Mahonin ae689c
Ivan Mahonin ae689c
def join(*args): return '/'.join(a for a in args if a)
Ivan Mahonin ae689c
def isdir(path): return not os.path.islink(path) and os.path.isdir(path)
Ivan Mahonin ae689c
def isfile(path): return not os.path.islink(path) and os.path.isfile(path)
Ivan Mahonin ae689c
Ivan Mahonin ae689c
def nameext(name):
Ivan Mahonin ae689c
    name = os.path.basename(name)
Ivan Mahonin ae689c
    i = name.rfind('.')
Ivan Mahonin ae689c
    return (name[:i], name[i+1:].lower()) if i >= 0 else (name, '')
Ivan Mahonin ae689c
def remove(path):
Ivan Mahonin ae689c
    if os.path.isfile(path) or os.path.islink(path):
Ivan Mahonin ae689c
        print('remove file:', path)
Ivan Mahonin ae689c
        os.remove(path)
Ivan Mahonin ae689c
    if os.path.isdir(path):
Ivan Mahonin ae689c
        for f in os.listdir(path): remove(join(path, f))
Ivan Mahonin ae689c
        print('remove dir:', path)
Ivan Mahonin ae689c
        os.rmdir(path)
Ivan Mahonin ae689c
def mkdirfor(path):
Ivan Mahonin ae689c
    os.makedirs(os.path.dirname(path), exist_ok = True)
Ivan Mahonin ae689c
def touch(path, msg = True):
Ivan Mahonin ae689c
    if msg: print('create file:', path)
Ivan Mahonin ae689c
    assert(not path in files)
Ivan Mahonin ae689c
    mkdirfor(path)
Ivan Mahonin ae689c
    files.add(path)
Ivan Mahonin ae689c
    return path
Ivan Mahonin ae689c
def link(src, dst):
Ivan Mahonin ae689c
    touch(dst, False)
Ivan Mahonin ae689c
    if not force and isfile(dst) and os.path.samefile(src, dst): return
Ivan Mahonin ae689c
    remove(dst)
Ivan Mahonin ae689c
    print('create link:', dst, '(' + src + ')')
Ivan Mahonin ae689c
    os.link(src, dst, follow_symlinks = False)
Ivan Mahonin ae689c
def clean(path, root = True):
Ivan Mahonin ae689c
    if os.path.islink(path) or (os.path.isfile(path) and not path in files):
Ivan Mahonin ae689c
        print('remove file:', path)
Ivan Mahonin ae689c
        os.remove(path)
Ivan Mahonin ae689c
    elif os.path.isdir(path):
Ivan Mahonin ae689c
        for f in os.listdir(path): clean(join(path, f), False)
Ivan Mahonin ae689c
        if not root and not os.listdir(path):
Ivan Mahonin ae689c
            print('remove dir:', path)
Ivan Mahonin ae689c
            os.rmdir(path)
Ivan Mahonin cad502
def mangle(path):
Ivan Mahonin cad502
    return base64.b64encode(hashlib.sha256((config.salt + ':' + path).encode()).digest()).decode()[:20].replace('/', '_').replace('+', '-')
Ivan Mahonin ae689c
Ivan Mahonin ae689c
Ivan Mahonin ae689c
class Node:
Ivan Mahonin ae689c
    def __init__(self, parent, src, name, hide = False):
Ivan Mahonin ae689c
        #print('node:', src)
Ivan Mahonin ae689c
        self.parent = parent
Ivan Mahonin ae689c
        self.src = src
Ivan Mahonin ae689c
        self.name = name
Ivan Mahonin ae689c
        self.title = name if parent else config.title
Ivan Mahonin ae689c
        self.isdir = isdir(src)
Ivan Mahonin ae689c
        self.ext = '' if self.isdir else nameext(name)[1]
Ivan Mahonin ae689c
        self.path = join(parent.path, name) if parent else name
Ivan Mahonin ae689c
        self.date = os.path.getmtime(src)
Ivan Mahonin cad502
        self.mangle = config.mangle + mangle(self.path) + '.' + self.ext
Ivan Mahonin ae689c
        self.hide = hide or (parent and parent.hide)
root 8883a2
        self.url = join((parent.url if self.parent else config.url), urllib.parse.quote(self.name))
Ivan Mahonin ae689c
        self.mangleurl = join(config.url, self.mangle)
Ivan Mahonin ae689c
        self.hideurl = join(config.hideurl, self.path) if self.isdir else (self.mangleurl if self.hide else self.url)
Ivan Mahonin ae689c
        self.thumburl = None
Ivan Mahonin ae689c
        self.thumbs = parent.thumbs if parent else {}
Ivan Mahonin ae689c
        self.subs = {}
Ivan Mahonin ae689c
        if parent: parent.touchdate(self.date); parent.subs[name] = self
Ivan Mahonin ae689c
Ivan Mahonin ae689c
    def touchdate(self, date):
Ivan Mahonin ae689c
        if self.date >= date: return
Ivan Mahonin ae689c
        self.date = date
Ivan Mahonin ae689c
        if self.parent: self.parent.touchdate(date)
Ivan Mahonin ae689c
Ivan Mahonin ae689c
    def scan(self):
Ivan Mahonin ae689c
        if not self.isdir: return
Ivan Mahonin ae689c
Ivan Mahonin ae689c
        td = join(self.src, '.thumb')
Ivan Mahonin ae689c
        if isdir(td):
Ivan Mahonin ae689c
            for f in os.listdir(td):
Ivan Mahonin ae689c
                n, e = nameext(f); fn = join(td, f)
Ivan Mahonin ae689c
                if e in thumbext and isfile(fn): self.thumbs[None if n == '.' else n] = fn
Ivan Mahonin ae689c
Ivan Mahonin ae689c
        for f in os.listdir(self.src):
Ivan Mahonin ae689c
            fn = join(self.src, f)
Ivan Mahonin ae689c
            if f[:1] != '.' or not isfile(fn): continue
Ivan Mahonin ae689c
            n, e = nameext(f[1:])
Ivan Mahonin ae689c
            if not e in thumbext: continue
Ivan Mahonin ae689c
            n, e = nameext(n)
Ivan Mahonin ae689c
            if n == 'thumb': self.thumbs[e] = fn
Ivan Mahonin ae689c
            if n == 'thumb.': self.thumbs[None] = fn
Ivan Mahonin ae689c
Ivan Mahonin ae689c
        for f in os.listdir(self.src): self.sub(join(self.src, f), f, False)
Ivan Mahonin ae689c
        hd = join(self.src, '.hide')
Ivan Mahonin ae689c
        if isdir(hd):
Ivan Mahonin ae689c
            for f in os.listdir(hd): self.sub(join(hd, f), f, True)
Ivan Mahonin ae689c
Ivan Mahonin ae689c
    def sub(self, src, name, hide):
Ivan Mahonin ae689c
        n, e = nameext(name)
Ivan Mahonin ae689c
        if ( self.isdir
Ivan Mahonin ae689c
          and name
Ivan Mahonin ae689c
          and name not in self.subs
Ivan Mahonin ae689c
          and name[0] != '.'
Ivan Mahonin ae689c
          and not name.lower() in skipname
Ivan Mahonin ae689c
          and (self.parent or not name.lower().startswith(config.mangle))
Ivan Mahonin ae689c
          and nameext(n)[1] != 'thumb'
Ivan Mahonin ae689c
          and (isfile(src) or isdir(src)) ):
Ivan Mahonin ae689c
            Node(self, src, name, hide).scan()
Ivan Mahonin ae689c
Ivan Mahonin ae689c
    def makefile(self):
Ivan Mahonin ae689c
        if self.isdir: return
Ivan Mahonin ae689c
        link(self.src, join(config.dst, self.mangle))
Ivan Mahonin ae689c
        if not self.hide: link(self.src, join(config.dst, self.path))
Ivan Mahonin ae689c
Ivan Mahonin ae689c
    def makethumb(self):
4cc0a2
        if self.hide and not config.hidedst: return
Ivan Mahonin ae689c
        ts = None
Ivan Mahonin ae689c
        td = None
Ivan Mahonin ae689c
        dst = config.hidedst if self.hide else config.dst
root 8883a2
        url = self.hideurl if self.hide else self.url
Ivan Mahonin ae689c
        if self.isdir:
Ivan Mahonin ae689c
            dst = join(dst, self.path, '.thumb.')
root 8883a2
            url = join(url, '.thumb.')
Ivan Mahonin ae689c
        else:
Ivan Mahonin ae689c
            dst = join(dst, self.path + '.thumb.')
root 8883a2
            url += '.thumb.'
Ivan Mahonin ae689c
Ivan Mahonin ae689c
        for e in reversed(thumbext):
Ivan Mahonin ae689c
            if isfile(join(self.src, '.thumb.' + e)): ts = join(self.src, '.thumb.' + e)
Ivan Mahonin ae689c
        for e in reversed(thumbext):
Ivan Mahonin ae689c
            if isfile(self.src +  '.thumb.' + e): ts = self.src +  '.thumb.' + e
Ivan Mahonin ae689c
Ivan Mahonin ae689c
        if not ts and not self.isdir:
Ivan Mahonin ae689c
            self.path + '.thumb.'
Ivan Mahonin ae689c
            if not force:
Ivan Mahonin ae689c
                for e in reversed(thumbext):
Ivan Mahonin ae689c
                    if isfile(dst + e) and os.path.getmtime(dst + e) > self.date: td = dst + e
Ivan Mahonin ae689c
            if td: touch(td, False)
Ivan Mahonin ae689c
            else:
Ivan Mahonin ae689c
                mkdirfor(dst + thumbext[0])
Ivan Mahonin ae689c
                td = thumb.make(self.src, dst + thumbext[0]);
Ivan Mahonin ae689c
                if td: touch(td)
Ivan Mahonin ae689c
Ivan Mahonin ae689c
        if not ts and not td:
Ivan Mahonin ae689c
            if self.isdir: ts = self.thumbs.get(None)
Ivan Mahonin ae689c
            else: ts = self.thumbs.get(self.ext, self.thumbs.get(''))
Ivan Mahonin ae689c
        if not td and ts:
Ivan Mahonin ae689c
            td = dst + nameext(ts)[1]
Ivan Mahonin ae689c
            link(ts, td)
Ivan Mahonin ae689c
        if td: self.thumburl = url + nameext(td)[1]
Ivan Mahonin ae689c
Ivan Mahonin ae689c
    def makeinfo(self):
Ivan Mahonin ae689c
        return {
Ivan Mahonin ae689c
            'name':      self.name,
Ivan Mahonin ae689c
            'title':     self.title,
Ivan Mahonin ae689c
            'date':      time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(self.date)),
Ivan Mahonin ae689c
            'isdir':     self.isdir,
Ivan Mahonin ae689c
            'hide':      self.hide,
Ivan Mahonin ae689c
            'url':       self.url,
Ivan Mahonin ae689c
            'hideurl':   self.hideurl,
Ivan Mahonin ae689c
            'mangleurl': self.mangleurl,
Ivan Mahonin ae689c
            'thumburl':  self.thumburl }
Ivan Mahonin ae689c
Ivan Mahonin ae689c
    def makepath(self):
Ivan Mahonin ae689c
        r = []; p = ''; n = self
Ivan Mahonin ae689c
        while n: r.insert(0, {'name': n.name, 'title': n.title, 'url': p}); p = join(p, '..'); n = n.parent
Ivan Mahonin ae689c
        return r
Ivan Mahonin ae689c
Ivan Mahonin ae689c
    def makeinfoex(self, bydate):
Ivan Mahonin ae689c
        r = self.makeinfo()
Ivan Mahonin ae689c
        r['bydate'] = bydate
Ivan Mahonin ae689c
        r['path'] = self.makepath()
Ivan Mahonin ae689c
        r['tw'] = config.thumbw
Ivan Mahonin ae689c
        r['th'] = config.thumbh
Ivan Mahonin ae689c
        return r
Ivan Mahonin ae689c
Ivan Mahonin ae689c
    def makeindex(self, bydate):
Ivan Mahonin ae689c
        if not self.isdir or self.hide: return
Ivan Mahonin ae689c
        fn = join(config.dst, self.path, ('indexd.html' if bydate else 'index.html'))
Ivan Mahonin ae689c
        if not force and isfile(fn) and os.path.getmtime(fn) > self.date: touch(fn, False); return
Ivan Mahonin ae689c
        r = self.makeinfoex(bydate)
Ivan Mahonin ae689c
        r['subs'] = []
Ivan Mahonin ae689c
        for n in sorted(self.subs.values(), key=lambda n: (not n.isdir, (-n.date if bydate else n.name))):
Ivan Mahonin ae689c
            if not n.hide: r['subs'].append(n.makeinfo())
Ivan Mahonin ae689c
        with open(touch(fn), 'w') as f: tpl.write(f, r)
Ivan Mahonin ae689c
Ivan Mahonin ae689c
    def makehideindex(self, bydate):
4cc0a2
        if not self.isdir or not config.hidedst: return
Ivan Mahonin ae689c
        fn = join(config.hidedst, self.path, ('indexd.html' if bydate else 'index.html'))
Ivan Mahonin ae689c
        if not force and isfile(fn) and os.path.getmtime(fn) > self.date: touch(fn, False); return
Ivan Mahonin ae689c
        r = self.makeinfoex(bydate)
Ivan Mahonin ae689c
        r['subs'] = [ n.makeinfo() for n in sorted(self.subs.values(), key=lambda n: (not n.isdir, (-n.date if bydate else n.name), not n.hide)) ]
Ivan Mahonin ae689c
        with open(touch(fn), 'w') as f: hidetpl.write(f, r)
Ivan Mahonin ae689c
Ivan Mahonin ae689c
    def make(self):
Ivan Mahonin ae689c
        #print('make:', self.path)
Ivan Mahonin ae689c
        for n in self.subs.values(): n.make()
Ivan Mahonin ae689c
        self.makefile()
Ivan Mahonin ae689c
        self.makethumb()
Ivan Mahonin ae689c
        self.makeindex(False)
Ivan Mahonin ae689c
        self.makeindex(True)
Ivan Mahonin ae689c
        self.makehideindex(False)
Ivan Mahonin ae689c
        self.makehideindex(True)
Ivan Mahonin ae689c
Ivan Mahonin ae689c
Ivan Mahonin ae689c
#os.umask(0o0027)
Ivan Mahonin ae689c
force = len(sys.argv) > 1 and sys.argv[1] == 'force'
Ivan Mahonin ae689c
files = set()
Ivan Mahonin ae689c
Ivan Mahonin ae689c
tpl = template.TplLoader.load("tpl/index.tpl")
Ivan Mahonin ae689c
hidetpl = template.TplLoader.load("tpl/hide.tpl")
Ivan Mahonin ae689c
Ivan Mahonin ae689c
n = Node(None, config.src, '')
Ivan Mahonin ae689c
n.scan()
Ivan Mahonin 3913d5
if time.time() - n.date <= config.guardsecs:
Ivan Mahonin 3913d5
    print('files are being update just now')
Ivan Mahonin 3913d5
    exit(0)
Ivan Mahonin ae689c
n.make()
Ivan Mahonin ae689c
Ivan Mahonin ae689c
clean(config.dst)
4cc0a2
if config.hidedst: clean(config.hidedst)
Ivan Mahonin ae689c
Ivan Mahonin 3913d5
#print('done')
Ivan Mahonin ae689c