import freebsd scripts from misc/freebsd repository
authorSofian Brabez <sbz@6dev.net>
Wed, 27 May 2015 20:22:13 +0000 (22:22 +0200)
committersbz <sbz@6dev.net>
Wed, 27 May 2015 20:22:13 +0000 (22:22 +0200)
README.md [new file with mode: 0644]
getpatch [new file with mode: 0755]
getprs [new file with mode: 0755]
myportlint [new file with mode: 0755]
poudriere-runner [new file with mode: 0755]
up [new file with mode: 0755]
vimport [new file with mode: 0755]

diff --git a/README.md b/README.md
new file mode 100644 (file)
index 0000000..e38befe
--- /dev/null
+++ b/README.md
@@ -0,0 +1 @@
+# freebsd-maintainer-scripts
diff --git a/getpatch b/getpatch
new file mode 100755 (executable)
index 0000000..c813475
--- /dev/null
+++ b/getpatch
@@ -0,0 +1,196 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2012 Sofian Brabez <sbz@FreeBSD.org>
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+#
+# $FreeBSD: head/Tools/scripts/getpatch 362667 2014-07-23 12:14:32Z sbz $
+#
+# MAINTAINER=   sbz@FreeBSD.org
+
+import argparse
+import codecs
+import re
+import sys
+if sys.version_info.major == 3:
+    import urllib.request as urllib2
+else:
+    import urllib2
+
+"""
+FreeBSD getpatch handles Gnats and Bugzilla patch attachments
+"""
+
+class GetPatch(object):
+
+    def __init__(self, pr, category='ports'):
+        self.pr = pr
+        self.category = category
+        self.patchs = []
+        self.url = ""
+        self.patch = ""
+        self.output_stdout = False
+        self.default_locale = sys.getdefaultencoding()
+
+    def fetch(self, *largs, **kwargs):
+        raise NotImplementedError()
+
+    def write(self, filename, data):
+        if filename.endswith(('.patch', '.txt')):
+            filename = filename[:filename.rindex('.')]+'.diff'
+        f=codecs.open(filename, encoding=self.default_locale, mode='w')
+        f.write(data.decode(self.default_locale))
+        f.close()
+        self.out("[+] %s created" % filename)
+
+    def get(self,only_last=False, output_stdout=False):
+        self.output_stdout = output_stdout
+        self.fetch(self.pr, category=self.category)
+
+        if len(self.patchs) == 0:
+            self.out("[-] No patch found")
+            sys.exit(1)
+
+        if only_last:
+            self.patchs = [self.patchs.pop()]
+
+        for patch in self.patchs:
+            url = patch['url']
+            p = patch['name']
+
+            data = urllib2.urlopen(url).read()
+
+            if self.output_stdout:
+                sys.stdout.write(data.decode(self.default_locale))
+            else:
+                self.write(p, data)
+
+    def add_patch(self, url, name):
+        self.patchs.append({'url': url, 'name': name})
+
+    def out(self, s):
+        if not self.output_stdout:
+            print(s)
+
+class GnatsGetPatch(GetPatch):
+
+    URL_BASE = 'https://www.freebsd.org/cgi'
+    URL = '%s/query-pr.cgi?pr=' % URL_BASE
+    REGEX = r'<b>Download <a href="([^"]*)">([^<]*)</a>'
+
+    def __init__(self, pr, category):
+        GetPatch.__init__(self, pr, category)
+
+    def fetch(self, *largs, **kwargs):
+        category = kwargs['category']
+        target = ("%s/%s" % (category, self.pr), "%s" % self.pr)[category=='']
+        self.out("[+] Fetching patch for pr %s" % target)
+        pattern = re.compile(self.REGEX)
+        u = urllib2.urlopen(self.URL+'%s' % target)
+        data = u.read()
+        if data == None:
+            self.out("[-] No patch found")
+            sys.exit(1)
+
+        for patchs in re.findall(pattern, str(data)):
+            self.add_patch(patchs[0], patchs[1])
+
+class BzGetPatch(GetPatch):
+
+    URL_BASE= 'https://bugs.freebsd.org/bugzilla/'
+    URL_SHOW = '%s/show_bug.cgi?id=' % URL_BASE
+    REGEX_URL = r'<a href="([^<]+)">Details</a>'
+    REGEX = r'<div class="details">([^ ]+) \(text/plain(?:; charset=[-\w]+)?\)'
+
+    def __init__(self, pr, category):
+        GetPatch.__init__(self, pr, category)
+
+    def _get_patch_name(self, url):
+        match = re.search(self.REGEX, urllib2.urlopen(url).read())
+        if match is None:
+            return None
+        return match.group(1)
+
+    def _get_patch_urls(self, data):
+        patch_urls = {}
+        for url in re.findall(self.REGEX_URL, data):
+            url = '%s/%s' % (self.URL_BASE, url)
+            file_name = self._get_patch_name(url)
+            if file_name is None:
+                self.out("[-] Could not determine the patch file name in %s. "
+                         "Skipping." % url)
+                continue
+            download_url = url[:url.find('&')]
+            patch_urls[download_url] = file_name
+        return patch_urls
+
+    def fetch(self, *largs, **kwargs):
+        category = kwargs['category']
+        self.out("[+] Fetching patch for pr %s/%s" % (category, self.pr))
+        u = urllib2.urlopen(self.URL_SHOW+'%s' % self.pr)
+        data = u.read()
+
+        if data == None:
+            self.out("[-] No patch found")
+            sys.exit(1)
+
+        patch_urls = self._get_patch_urls(data)
+        if not patch_urls:
+            self.out("[-] No patch found")
+            sys.exit(1)
+
+        for url, file_name in patch_urls.iteritems():
+            self.add_patch(url, file_name)
+
+def main():
+
+    parser = argparse.ArgumentParser(
+            description='Gets patch attachments from a Bug Tracking System'
+    )
+    parser.add_argument('pr', metavar='pr', type=str, nargs=1,
+            help='Pr id number')
+    parser.add_argument('--mode', type=str, choices=['gnats','bz'],
+            default='bz', help='available modes to retrieve patch')
+    parser.add_argument('--last', action='store_true',
+            help='only retrieve the latest iteration of a patch')
+    parser.add_argument('--stdout', action='store_true',
+            help='dump patch on stdout')
+
+    if len(sys.argv) == 1:
+        parser.print_help()
+        sys.exit(1)
+
+    args = parser.parse_args()
+
+    category = ""
+    pr = str(args.pr[0])
+
+    if '/' in pr and pr is not None:
+        category, pr = pr.split('/')
+
+    Clazz = globals()['%sGetPatch' % args.mode.capitalize()]
+    gp = Clazz(pr, category)
+    gp.get(only_last=args.last, output_stdout=args.stdout)
+
+if __name__ == '__main__':
+    main()
diff --git a/getprs b/getprs
new file mode 100755 (executable)
index 0000000..62ba519
--- /dev/null
+++ b/getprs
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+: {USER:="sbz"}
+
+SCRIPT=${0##*/}
+SCRIPT_VERSION="1.0"
+
+PR_SUMMARY_URL="http://www.freebsd.org/cgi/query-pr-summary.cgi?responsible=${USER}"
+
+format_line() {
+       sed -n "s#^<tr class='o'><td>o</td><td>\([^<]*\)</td><td><a href='/cgi/query-pr.cgi?pr=[^']*'>\([^<]*\)</a></td><td>[^<]*</td><td>\([^<]*\)</td></tr>#\1 | \2  | \3#p"
+}
+
+fetch_prs() {
+       HTTP_USER_AGENT="${SCRIPT}/${SCRIPT_VERSION}" fetch -q -o - ${PR_SUMMARY_URL} 2>&1 | format_line
+}
+
+fetch_prs
diff --git a/myportlint b/myportlint
new file mode 100755 (executable)
index 0000000..a4c1e55
--- /dev/null
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+: ${PORTSDIR:="/usr/ports"}
+: ${MAINTAINER:="${1:-${USER}}@FreeBSD.org"}
+
+PORTLINT_FLAGS="-Abct${PORTLINT_FLAGS}"
+SCRIPT=$0
+SCRIPT=${SCRIPT##*/}
+
+ports_by_maintainer() {
+       ports=`make -C ${PORTSDIR} search key=${MAINTAINER} display="path"|sed -n "s#Path:.*${PORTSDIR}\(.*\)#\1#p"`
+}
+
+cleanup() {
+       [ -f /tmp/${SCRIPT}.log ] && rm -f /tmp/${SCRIPT}.log
+}
+
+main() {
+       cleanup
+
+       ports_by_maintainer
+
+       for port in $ports
+       do
+               cd "${PORTSDIR}/$port"
+               echo "===> portlint output on port $port ==" >> /tmp/${SCRIPT}.log
+               portlint ${PORTLINT_FLAGS} >> /tmp/${SCRIPT}.log
+               echo -e "<=== portlint output ==\n" >> /tmp/${SCRIPT}.log
+               cd -
+       done
+}
+
+main
diff --git a/poudriere-runner b/poudriere-runner
new file mode 100755 (executable)
index 0000000..b8856f9
--- /dev/null
@@ -0,0 +1,194 @@
+#!/usr/bin/env python3
+
+"""
+sbz poudriere runner
+"""
+
+import argparse
+import configparser
+import os
+import shlex
+import subprocess
+import sys
+
+LOCALBASE="/usr/local"
+PORTSDIR="/usr/ports" or os.environ['PORTSDIR']
+
+default_conf="""
+[default]
+debug=False
+disk_path=%s/zfsfs
+jails=10i386,10amd64,84amd64,84i386,93amd64,93i386
+jails_disabled=
+mdconfig_cmd=mdconfig -f
+cpuset=True
+cpuset_cmd=cpuset -c -l 0-3
+
+[84i386]
+name=84i386
+arch=i386
+version=8.4-RELEASE
+
+[84amd64]
+name=84amd64
+arch=amd64
+version=8.4-RELEASE
+
+[93i386]
+name=93i386
+arch=i386
+version=9.3-RELEASE
+
+[93amd64]
+name=93amd64
+arch=amd64
+version=9.3-RELEASE
+
+[10i386]
+name=10i386
+arch=i386
+version=10.1-RELEASE
+
+[10amd64]
+name=10amd64
+arch=amd64
+version=10.1-RELEASE
+""" % LOCALBASE
+
+class Prunner(object):
+    def __init__(self):
+        loadDisk()
+        self._jails = loadJails()
+        self._port_trees = []
+        self.is_setup = False
+        if not self.is_setup:
+            self.setUp()
+
+    def setUp(self, **params):
+        if debug:
+            print("setup...")
+        #TODO: test if exists to avoid recreating jails
+        params['host'] = 'FREEBSD_HOST=ftp.fr.freebsd.org'
+        for jail in self._jails:
+            cmd = "poudriere jail -c -j {0} -a {1} -v {2} {3}".format(
+                jail['name'],
+                jail['arch'],
+                jail['version'],
+                params['host'])
+            if debug:
+                print("exec: {0}".format(cmd))
+            sudo(cmd)
+
+    def tearDown(self):
+        #TODO: kill jail (stop them)
+        for jail in self._jails:
+            cmd = "poudriere jail -k -j {0}".format(jail['name'])
+            if debug:
+                print("exec: {0}".format(cmd))
+            sudo(cmd)
+
+    @property
+    def jails(self):
+       return self._jails
+
+    @property
+    def port_trees(self):
+       return self._port_trees
+
+    def testPort(self, origin, port_tree="portsdir"):
+        for jail in self._jails:
+            if debug:
+                print("testport o: {0}: j: {1}".format(origin, jail['name']))
+            cmd = "poudriere testport -o {0} -j {1} -p {2} -n".format(
+                    origin,
+                    jail['name'], port_tree)
+            out, err = sudo(cmd)
+            if debug:
+                print("j: {0}, out: {1}, err: {2}".format(jail['name'], out,
+                    err))
+
+    def testAll(self):
+        for o in self.sbzports():
+            self.test_port(o)
+
+    def bulk(self, jail):
+        raise NotImplemented("not available yet")
+
+    def sbzPorts(self):
+        #TODO: don't rely on sbzports file
+        ports = open('{0}/sbzports'.format(os.path.expanduser('~%s' %
+            os.environ['USER'])), 'r').read().strip().split('\n')
+        return ports
+
+    def __del__(self):
+        if debug:
+            print("teardown()...")
+        if hasattr(self, 'is_setup') and self.is_setup:
+            self.tearDown()
+
+def run(command, use_sudo=False):
+    cpuset_cmd = cfg.get('default', 'cpuset_cmd')
+    use_cpuset = cfg.get('default', 'cpuset')
+    if use_cpuset:
+        command = "%s %s" % (cpuset_cmd, command)
+
+    command = "sudo %s" % command if use_sudo else command
+    p = subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE)
+    stdout, stderr = p.communicate()
+    p.wait()
+    return stdout.strip(), stderr
+
+def sudo(command):
+    return run(command, use_sudo=True)
+
+def loadDisk():
+    mdconfig_cmd = cfg.get('default', 'mdconfig_cmd')
+    disk_path = cfg.get('default', 'disk_path')
+    if not os.path.exists('/dev/md0'):
+        out, err = sudo("{0} {1}".format(mdconfig_cmd, disk_path))
+        if err is not None:
+            print("Error: {0}".format(out))
+            sys.exit(1)
+    return True
+
+def loadJails():
+    jails = []
+    existing_jails = cfg.get('default', 'jails').split(',')
+    try:
+        existing_jails.remove(cfg.get('default', 'jails_disabled'))
+    except:
+        pass
+
+    for jail in existing_jails:
+        name = cfg.get(jail, 'name')
+        arch = cfg.get(jail, 'arch')
+        version = cfg.get(jail, 'version')
+        jails.append({'name': name, 'arch': arch, 'version': version})
+
+    return jails
+
+def isValidOrigin(origin):
+    abspath = "{0}/{1}".format(PORTSDIR, origin)
+    return os.path.isdir(abspath)
+
+def main():
+    global cfg, debug
+    cfg = configparser.ConfigParser()
+    cfg.read_string(default_conf)
+    debug = cfg.get('default', 'debug')
+    if debug:
+        print(cfg.get('default', 'disk_path'))
+
+    runner = Prunner()
+    if len(sys.argv) == 1:
+        cwd = os.getcwd()
+        origin = "{0}/{1}".format(cwd.split('/')[-2], cwd.split('/')[-1])
+    else:
+        origin = sys.argv[1]
+
+    if isValidOrigin(origin):
+        runner.testPort(origin)
+    #runner.testAll()
+
+if __name__ == '__main__':
+    main()
diff --git a/up b/up
new file mode 100755 (executable)
index 0000000..525cdf5
--- /dev/null
+++ b/up
@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+
+"""
+One updater script to pull them all
+"""
+
+import configparser
+import io
+import os
+import shlex
+import subprocess
+import sys
+
+user = os.environ['USER']
+pwd = os.path.expanduser('~{0}'.format(user) or '/home/{0}'.format(user))
+
+config = """
+[default]
+scm=git
+git_update_cmd=git pull
+git_clone_cmd=git clone
+svn_update_cmd=svn update
+svn_clone_cmd=svn checkout
+
+[freebsd-src]
+scm=svn
+remote=svn+ssh://{1}@svn.freebsd.org/base/head
+local={0}/freebsd/svn/src
+
+[freebsd-ports]
+scm=svn
+remote=svn+ssh://{1}@svn.freebsd.org/ports/head
+local={0}/freebsd/svn/ports
+
+[freebsd-doc]
+scm=svn
+remote=svn+ssh://{1}@svn.freebsd.org/doc/head
+local={0}/freebsd/svn/doc
+
+[netbsd-src]
+remote=http://git.edgebsd.org/EdgeBSD/netbsd-src.git
+local={0}/netbsd/git/src
+
+[netbsd-pkgsrc]
+remote=http://git.edgebsd.org/EdgeBSD/netbsd-pkgsrc.git
+local={0}/netbsd/git/pkgsrc
+
+[openbsd-src]
+remote=git://anoncvs.estpak.ee/openbsd-src
+local={0}/openbsd/git/src
+""".format(pwd, user)
+
+cfg = configparser.ConfigParser()
+cfg.read_string(config)
+
+def call(cmd):
+    subprocess.call(shlex.split(cmd))
+
+class UpdaterFactory(object):
+
+    class BaseUpdater(object):
+        def __init__(self):
+            self.scm = "none"
+            self.already_cloned = False
+
+        def setCloned(self):
+            self.already_clone = True
+
+        def isCloned(self):
+            return self.already_cloned
+
+        def up(self): raise NotImplemented("up() method to implement")
+
+        def clone(self): raise NotImplemented("clone() method to implement")
+
+    class SVNUpdater(BaseUpdater):
+        def __init__(self, p):
+            super().__init__()
+            self.p = p
+            self.scm = 'svn'
+
+        def up(self):
+            os.chdir(self.p.get('local'))
+            cmd = cfg.get('default', 'svn_update_cmd')
+            call(cmd)
+
+        def clone(self):
+            cmd = cfg.get('default', 'svn_clone_cmd')
+            rc = call('svn checkout {0} {1}'.format(self.p.get('remote'), self.p.get('local')))
+            if rc == 0:
+                self.setCloned()
+
+    class GitUpdater(BaseUpdater):
+        def __init__(self, p):
+            super().__init__()
+            self.p = p
+            self.scm = 'git'
+
+        def up(self):
+            os.chdir(self.p.get('local'))
+            cmd = cfg.get('default', 'git_update_cmd')
+            call(cmd)
+
+        def clone(self):
+            cmd = cfg.get('default', 'git_clone_cmd')
+            rc = call('{0} {1} {2}'.format(cmd , self.p.get('remote'), self.p.get('local')))
+            if rc == 0:
+                self.setCloned()
+
+    @classmethod
+    def fromdict(cls, p):
+        scm_type = p.get('scm')
+        if scm_type == 'git':
+            return cls.GitUpdater(p)
+        elif scm_type == 'svn':
+            return cls.SVNUpdater(p)
+        else:
+            raise Exception("scm_type '{0}' unknown".format(scm_type))
+
+class Config(object):
+    def __init__(self, cp):
+        self.cp = cp
+
+    def getSection(self, s, k):
+        if self.cp is None:
+            return ""
+        return self.cp._sections[s].get(k)
+
+    def getProjects(self):
+        projects = []
+        for s in self.cp._sections:
+            if s == 'default':
+                continue
+            remote = self.getSection(s, 'remote')
+            local = self.getSection(s, 'local')
+            scm = self.getSection(s, 'scm') or self.getSection('default', 'scm')
+            projects.append(
+                {'name': s, 'remote': remote, 'local': local, 'scm': scm}
+            )
+
+        return projects
+
+if __name__ == '__main__':
+
+    for p in Config(cfg).getProjects():
+        u = UpdaterFactory.fromdict(p)
+        if not u.isCloned():
+            u.clone()
+        else:
+            u.up()
diff --git a/vimport b/vimport
new file mode 100755 (executable)
index 0000000..1189957
--- /dev/null
+++ b/vimport
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+: ${PORTSDIR:="/usr/ports"}
+
+cd ${PORTSDIR}
+vim -p $(cd ${PORTSDIR}; psearch -m ${USER}|cut -d' ' -f1|xargs -I {} echo {}/Makefile)