diff options
Diffstat (limited to 'util')
| -rw-r--r-- | util/.marauder/marauder/__init__.py | 4 | ||||
| -rw-r--r-- | util/.marauder/marauder/install.py | 200 | ||||
| -rw-r--r-- | util/.marauder/marauder/parser.py | 277 | ||||
| -rw-r--r-- | util/.marauder/marauder/version.py | 40 | ||||
| -rwxr-xr-x | util/brew-convert.py | 225 |
5 files changed, 746 insertions, 0 deletions
diff --git a/util/.marauder/marauder/__init__.py b/util/.marauder/marauder/__init__.py new file mode 100644 index 000000000..379afe39b --- /dev/null +++ b/util/.marauder/marauder/__init__.py @@ -0,0 +1,4 @@ +__version__ = '2.0' +from .parser import parse +from .version import version +from .install import makesh
\ No newline at end of file diff --git a/util/.marauder/marauder/install.py b/util/.marauder/marauder/install.py new file mode 100644 index 000000000..3c7b20405 --- /dev/null +++ b/util/.marauder/marauder/install.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +import re +import csv +import sys + + +# FIXME 18 Errors - 18/02 + +def _variable(word): + + def ENV(): + # Easier to parse & handle this way. + ENV.cc = '${PKG_TARG}-clang' + ENV.ld = '${PKG_TARG}-ld' + ENV.cxx = '${PKG_TARG}-clang++' + ENV.cflags = '${CFLAGS}' + ENV.ldflags = '${LDFLAGS}' + ENV.cxxflags = '${CXXFLAGS}' + ENV.cppflags = '${CPPFLAGS}' + ENV() + + prefix = '${PKG_TAPF}' + man = '${PKG_TAPF}' + '/share/man' + pkgshare = '${PKG_TAPF}' + '/share' + lib = '${PKG_TAPF}' + '/lib' + share = '${PKG_TAPF}' + '/share' + + sysconfig = '/etc' + bin = '${PKG_TAPF}' + 'bin' + + # include = '$SDK' + '/usr/include' + + b = ['ENV', 'prefix', 'man', 'pkgshare', 'lib', 'include'] + + w = ''.join(word) + try: + if not any([x for x in b if w.startswith(x)]): + return False # Prevent code execution + word = eval(w) + except NameError: + return False + except AttributeError: +# print(f'WARNING: {w} needs to be defined.', file=sys.stderr) + # raise + return False + + return word + + + +def _system(): + """ Handles strings like: system "make", "install """ + + def _configure(): + nonlocal l + banned = [#'--disable-debug', + '--prefix=', + '--man=', + '--mandir=', + '--localstatedir=', + '--sysconfdir=', + '--infodir=', + '--datadir' + '--disable-dependency-tracking', + '--disable-debug'] + regex = re.compile(f"{'|'.join(banned)}*") + track = '' + for x in l: + if not any([a for a in x.split() if regex.search(a)]): + track += f' {x}' + + l = track.strip() + try: + conf = re.search('(^.+/configure)', l).group(1) + except AttributeError: + # print("ERROR: configure - AttributeError - NoneType has no attribute 'group'", file=sys.stderr) + conf = l[0] + if conf == './configure': + l = l.replace(conf, f"pkg:configure") + else: + l = l.replace(conf, f"PKG_CONF={conf} pkg:configure") + + def _make(): + nonlocal l + l = ' '.join(l) + l = l.replace('install', 'DESTDIR=${PKG_DEST} install', 1) +# if == 'make': # if the line is just `make` +# l = l.replace('make', 'pkg:make') # Hand it to pkg:make for the args + + def _cmake(): + nonlocal l + l = ' '.join(l) + l = l.replace(r'*std_cmake_args', r'-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}') + + + l = re.sub(r'^system', '', g_line).strip() + l = list(csv.reader([l], quotechar='"', skipinitialspace=True))[0] + + # Convert #{variable} to a bash variable. + variable = [k for k, p in enumerate(l) if re.findall(r'#{(?:[A-Za-z0-9.]+)}', p)] + if variable: + for p in variable: + n = re.findall(r'#{([A-Za-z0-9.]+)}', l[p]) + k = _variable(n) + if not k: + continue # _variable() returned False for some reason. + l[p] = l[p].replace(f"#{{{''.join(n)}}}", f'{k}') + + if 'configure' in l[0]: + _configure() + elif 'cmake' in l[0]: + _cmake() + elif 'make' in l[0]: + _make() + else: + l = ' '.join(l) + + return l + + +def _inreplace(): + # TODO: inreplace |s|, let replace be a bash function, so remove any shell characters. + return -1 + +def _clean(file): + """ Cleans up input for parsing """ + file = iter(file) + + def list(): + # FIXME: 2 Formulas will error out here - ['inreplace %w[configure Makefile], for example. + nonlocal x + if re.search(r'%[A-Za-z]\[', x): + track = x + if x.endswith("]") or '].each' in x: + return track + else: + while True: + l = next(file) + track += f'{l}' + if re.search(r'^\]\.?', l): + break + track += ' ' + return track + return False + + def newlines(): + nonlocal x + if not x.endswith(','): + return False + line = x + l = x + while True: + if not line.endswith(','): + break + line = next(file) + l += f' {line}' + return l + + out = [] + for x in file: + _list = list() + if _list: + out += [_list] + else: + _newline = newlines() + if _newline: + out += [_newline] + else: + out += [x] + return out + + +def makesh(file=None): + try: + data = _clean(file) + except StopIteration: + # print(f'ERROR: parse - StopIteration - {file}', file=sys.stderr) + return None # Error + global g_line # Global line + n = [] + for g_line in data: + if g_line.endswith('if build.head?'): + continue # Doesn't support head. + elif g_line.startswith("system"): + n += [_system()] + elif g_line.startswith("mkdir"): + try: + cd = re.findall(r'\"(.+?)\"', g_line)[0] + except IndexError: + # TODO - Fix index error + # print("ERROR: parse - IndexError - cd regex failed", file=sys.stderr) + n += [g_line] + continue + n += [re.sub('do$', f'&& cd {cd}', g_line)] + elif g_line.startswith("cd"): + n += [re.sub('do$', '', g_line)] + else: + n += [g_line] + + return n diff --git a/util/.marauder/marauder/parser.py b/util/.marauder/marauder/parser.py new file mode 100644 index 000000000..1d5d2fdd1 --- /dev/null +++ b/util/.marauder/marauder/parser.py @@ -0,0 +1,277 @@ +import re +import sys + +keywords = re.compile(r"^(desc|homepage|url|mirror|bottle|patch|def|resource|depends_on|conflicts_with|__END__)") + +regex_quoted = re.compile(r'["\'](.*?)["\']') +regex_resource = re.compile(r'resource "(.*?)" do') + +def _reset(): + global result + result = { + "name": None, + "description": None, + "url": None, + "mirror": None, + "homepage": None, + "depends": [], + "resource": [], + "conflicts": [], + "patches": [], + "install": [] + } + + +def _error(msg, err): + print("ERROR: {} - {}".format(msg, err), file=sys.stderr) + + +def _name(): + t = _clean(next(line)) + if 'class' not in t: + return # Might be text at the top of file. + try: + t = t.split()[1] + except IndexError as err: + if "class" in t: + _error("_name()", err) + raise err + else: + pass + result['name'] = t + + +def _patch(): + url = None + strip = None + data = None + + if x.endswith("__END__"): + data = [] + while True: + try: + data.append(next(line)) + except StopIteration: + break + d = {"url": url, "data": data} + result["patches"].append(d) + return + try: + t = re.match(r'(?:patch) :p?(\d|DATA)', x).groups(0)[0] + except AttributeError: + """ `patch do` would be the case here, assume strip=1 """ + t = "1" + if "DATA" in t: + """ We'll get called later when __END__ appears """ + # print("Expecting DATA at EOF.") # Debug line + return + else: + strip = t + while True: + a = _clean(next(line)) + # print("----> {}".format(a)) # Debug + if a.startswith("url"): + try: + url = regex_quoted.findall(a)[0] + except IndexError as err: + _error("IndexError in _patch()", err) + return -1 + if a.startswith("end"): + break + d = {"url": url, "strip": strip} + result["patches"].append(d) + + +def _resource(): + name = None + url = None + if not regex_resource.match(x): + return # TEMP, CLEAN UP + # Resource "XX" do + try: + name = regex_quoted.findall(x)[0] + except IndexError as err: + _error("IndexError in _resource() (name)", err) + return -1 + while True: + try: + t = _clean(next(line)) + except StopIteration as err: + _error("StopIteration in _resource()", err) + break + # print(t) # debug + if t.startswith("url"): + try: + url = regex_quoted.findall(t)[0] + except IndexError as err: + _error("IndexError in _resource() (url)", err) + return -1 + if t.startswith("end"): + break + d = {"name": name, "url": url} + result["resource"].append(d) + return + + +def _homepage(): + try: + homepage = regex_quoted.findall(x)[0] + result['homepage'] = homepage + return + except IndexError as err: + _error("IndexError in _homepage()", err) + return -1 + + +def _url(): + try: + url = regex_quoted.findall(x)[0] + if not result['url']: + result['url'] = url + return None + else: + return url + except IndexError as err: + _error("IndexError in _url()", err) + return None + + +def _mirror(): + mirror = _url() + if mirror: + result['mirror'] = mirror + else: + return -1 + + +def _bottle(): + return -1 + + +def _description(): + try: + description = regex_quoted.findall(_x)[0] + result['description'] = description + except IndexError as err: + _error("IndexError in _description()", err) + return -1 + return + + +def _clean(_line): + """ Strips new lines and comments""" + _line = _line.strip() + if "# " in _line: + _line = _line[:_line.index('#')] + return _line + + +def _depends(): + name = None + build = False # Build Dependency + try: + a = None + try: + a = re.findall(r'depends_on :(\S*)', x)[0] + except IndexError: + pass + if a: + name = a + else: + name = regex_quoted.findall(x)[0] + except IndexError as err: + _error("IndexError in _depends(): line {}".format(x), err) + return -1 + if x.endswith(":build"): + build = True + d = {'depend': name, 'build-depend': build} + result["depends"].append(d) + return + + +def _conflicts(): + name = None + reason = None + try: + if ':because ' in x: + t = x.split(':because ')[1] + reason = regex_quoted.findall(t)[0] + name = regex_quoted.findall(x)[0] + except IndexError as err: + _error("IndexError in _conflicts()", err) + return -1 + d = {'conflict': name, 'reason': reason} + result['conflicts'].append(d) + return + +def _install(): + grammar = re.compile(r'( do|if )') + l = [] + c = 0 + while True: + a = _clean(next(line)) + if grammar.match(a): + c += 1 + elif a.startswith("end"): + if c <= 0: + break + else: + c -= 1 + l.append(a) + result["install"] = l + return + + +def _def(): + if "install" in x: + _install() + return -1 + + +def _keywords(word): + if word.startswith("desc") and not result['description']: + _description() + elif word.startswith("homepage") and not result['homepage']: + _homepage() + elif word.startswith("url") and not result['url']: + _url() + elif word.startswith("mirror") and not result['mirror']: + _mirror() + elif word.startswith("bottle"): + _bottle() + elif word.startswith("patch") or word.startswith("__END__"): + _patch() + elif word.startswith("resource"): + _resource() + elif word.startswith("depends_on"): + _depends() + elif word.startswith("conflicts_with"): + _conflicts() + elif word.startswith("def"): + _def() + else: + return -1 + + +def _parse(file): + """ Reads the data """ + global line, x, _x + # a = 0 # debug + with open(file) as line: + line = iter(line) + while not result["name"]: + _name() + for x in line: + _x = x # Temp workaround for "#" in descriptions + x = _clean(x) + # a += 1 # debug + # print("{} {}".format(a, x)) # Debug + if keywords.match(x): + # print("keywords match ({}) found! {}".format(keywords.search(x).group(1), x)) # DEBUG + _keywords(keywords.search(x).group(1)) + + +def parse(file): + _reset() + _parse(file) + return result diff --git a/util/.marauder/marauder/version.py b/util/.marauder/marauder/version.py new file mode 100644 index 000000000..6ad09d9fe --- /dev/null +++ b/util/.marauder/marauder/version.py @@ -0,0 +1,40 @@ +import os +from io import BytesIO +import re +from urllib.request import urlopen +import sys + +default = re.compile(r'(?:([0-9]+(?:[.-_][0-9]+)+))') # Version() + +""" Takes URL string and guesses version """ +def version(url): + filename = os.path.basename(url) + version = default.findall(filename) + if not version: + x = os.path.basename(os.path.dirname(url)) + version = default.findall(x) # Go up one directory and try that. + if version: + return version[0] + version = re.findall(r'\S*?(:?[0-9-]+)', filename) # v40 or program-83 for example. + if version: + return version[0] + if url.endswith(".git"): + return None # FIXME: this is too slow. + # print(f"Finding version for {url}", file=sys.stderr) + latest_release = os.popen( + f"git ls-remote --tags {url} 2>/dev/null | sort -Vk2 | awk 'END{{ print $2 }}' | grep -oE '[0-9]+(\\.[0-9]+)+'" + ).read().strip() + latest_commit = os.popen( + f"git ls-remote --tags {url} 2>/dev/null | sort -Vk2 | awk 'END{{ print $1 }}'" + ).read().strip() + if latest_release and latest_commit: + version = f"{latest_release}-git-{latest_commit[:8]}" + elif latest_commit: + version = f"1.0-git-{latest_commit[:8]}" + else: + return None + return version + + if not version: # If still no result. + return None + return version[0] diff --git a/util/brew-convert.py b/util/brew-convert.py new file mode 100755 index 000000000..f027ad54b --- /dev/null +++ b/util/brew-convert.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +import sys +import os +import json +import glob +import zipfile +import tempfile +import argparse +from urllib.request import urlopen +from io import BytesIO + +sys.path.append(os.path.split(os.path.abspath(__file__))[0] + '/.marauder') +import marauder as m + +# Global variables: +# Swapping for linux-brew might work. +url = 'https://github.com/Homebrew/homebrew-core/archive/master.zip' +# Use this string as the default version +_version = "1.0-unknown" +# Pick the hidden dot-file that holds json information. +_dotfile = "beer" + + +def download(): + """ Downloads and extracts the url """ + print(f"Downloading {url}", file=sys.stderr) + + global extract + dl = urlopen(url) + z = zipfile.ZipFile(BytesIO(dl.read())) + extract = os.path.commonprefix(z.namelist()) + "Formula" + + for file in z.namelist(): + if file.startswith(extract): + z.extract(file) + + +def convert(create=False): + """ General purpose, converts the formulas """ + global extract + + def creator(data): + """ Creates/write out the formula to the build-system """ + nonlocal file + + def skip(): + """ Returns true if formula has any unwanted dependencies """ + nonlocal data + s = ['python@2', + 'python', + 'go', + 'mono', + 'ruby', + 'x11', + 'xcode', + 'java', + 'rust', + 'osxfuse', + 'gtk+3', + 'zsh', + 'perl', + 'node', + 'cabal-install', + 'ghc', + 'ant'] + + for d in data['depends']: + if any(d['depend'] in depend for depend in s): + return True + return False + + if skip(): + return None + elif data['url'].endswith('.git') or 'svn.' in os.path.dirname(data['url']): + return None # example.com/foo.git; svn.example.com/trunk + + data['version'] = m.version(data['url']) or _version # <--- or default + data['install'] = [line for line in data['install'] if line != ''] # Strips lines + data['file'] = os.path.basename(file) # foobar.json | The original formula + + # print(f"{data['name']}\t{data['version']}\t{data['url']}", file=sys.stderr) + + # Create directories + n = data['name'].lower() # Name + try: + os.makedirs(f"{n}/_metadata") + except FileExistsError as e: + print(f"Directory {data['name']} already exists! Skipping... ({e}) {data['file']}", file=sys.stderr) + return -1 + with open(f"{n}/_metadata/name", 'w') as f_name, \ + open(f"{n}/_metadata/description", 'w') as f_description, \ + open(f"{n}/_metadata/version", 'w') as f_version, \ + open(f"{n}/_metadata/homepage", 'w') as f_homepage, \ + open(f"{n}/.{_dotfile}", 'w') as f_dotfile, \ + open(f"{n}/.make.sh-auto", 'w') as f_make, \ + os.fdopen(os.open(f"{n}/download.sh", os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o755), 'w') as f_download: + # _metadata + print(f"{data['name']}", file=f_name) # Use print because it appends a newline. + print(f"{data['description']}", file=f_description) + print(f"{data['version']}", file=f_version) + print(f"{data['homepage']}", file=f_homepage) + # JSON information file for whatever + json.dump(data, f_dotfile, indent=4) + # Create a script to download the file. + f_download.write(f"wget {data['url']}") + if data['mirror']: + f_download.write(f" || wget {data['mirror']}") + if data['patches']: + with os.fdopen(os.open(f"{n}/patches.sh", os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o755), 'w') as f_patches: + for p in data['patches']: + if not p['url']: # There's data in this entry. + try: + assert p['data'] # But it might be a parsing error. + + print("echo 'Creating brew-patch.diff'", file=f_patches) + print("cat << EOF >> brew-patch.diff", file=f_patches) + for l in p['data']: + f_patches.write(l) + print("EOF", file=f_patches) + except KeyError: + pass + else: # not p['url'] + print(f"wget {p['url']}", file=f_patches) + # // if data['patches'] + if data['install']: + try: + print("pkg:setup", file=f_make) + for l in m.makesh(data['install']): + print(f"{l}", file=f_make) + except TypeError: # FIXME + os.remove(f"{n}/.make.sh-auto") + pass + + # TODO: + # -> Parse `def install` + + def updater(dotfile, data, file): + """ Updates a formula """ + if dotfile['url'] == data['url']: + # print(f"{dotfile['name']} is up to date.", file=sys.stderr) + return 0 + # Version comes from the urls, so check the urls instead. + else: + print(f"{dotfile['name']} has to be updated.", file=sys.stdout) + + data['version'] = m.version(data['url']) or _version + data['install'] = [line for line in data['install'] if line != ''] + data['file'] = dotfile['file'] + + n = os.path.dirname(file) + with open(f"{n}/_metadata/version", 'w') as f_version, \ + open(f"{n}/.{_dotfile}", 'w') as f_dotfile, \ + os.fdopen(os.open(f"{n}/download.sh", os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o755), 'w') as f_download: + + print(f"{data['version']}", file=f_version) + json.dump(data, f_dotfile, indent=4) + f_download.write(f"wget {data['url']}") + if data['mirror']: + f_download.write(f" || wget {data['mirror']}") + try: # Just try to remove the old tarball. + os.remove(f"{n}/" + os.path.basename(dotfile['url'])) + except OSError: + pass + if data['patches']: + with os.fdopen(os.open(f"{n}/patches.sh", os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o755), 'w') as f_patches: + for p in data['patches']: + if not p['url']: + try: + assert p['data'] + print("echo 'Creating brew-patch.diff'", file=f_patches) + print("cat << EOF >> brew-patch.diff", file=f_patches) + for l in p['data']: + f_patches.write(l) + print("EOF", file=f_patches) + except KeyError: + pass + else: + print(f"wget {p['url']}", file=f_patches) + + + with tempfile.TemporaryDirectory() as t: + # `pushd` tmpdir + cwd = os.getcwd() + os.chdir(t) + # Grab master and extract + download() + assert extract + extract = os.path.realpath(extract) + # `popd` + os.chdir(cwd) + + if create: + for file in sorted(glob.glob(extract + '/*')): + creator(m.parse(file)) + else: + for file in sorted(glob.glob(f"*/.{_dotfile}")): + f = json.load(open(file)) + rb = f"{extract}/" + f['file'] + updater(f, m.parse(rb), file) + + +def parse_arg(argv): + parser = argparse.ArgumentParser(prog='Brew-Converter', description='Converts homebrew formulas to `pwd`. Be sure to cd before using this tool.') + group = parser.add_mutually_exclusive_group() + # Args + group.add_argument('--all', help='Create and parse all brew formulas', + action='store_true', default=False, dest='all') + group.add_argument('--upgrade', help='Upgrade all formulas present', + action='store_true', default=False, dest='upgrade') + results = parser.parse_args() + # Checking + if len(argv) <= 0: + parser.print_help() + sys.exit(1) + elif results.all: + convert(create=True) + elif results.upgrade: + convert() + else: + print(f"Something went wrong: {results}") + sys.exit(3) + + +if __name__ == '__main__': + parse_arg(sys.argv[1:]) |