Merge pull request #439 from urfave/migrators

Migrator touchup and testing

.travis.yml

@@ -10,12 +10,15 @@ go:
 - 1.6.2
 - master
 
+env: pip_install="pip install --user"
+
 matrix:
   allow_failures:
   - go: master
   include:
   - go: 1.6.2
     os: osx
+    env: pip_install="sudo pip install"
   - go: 1.1.2
     install: go get -v .
     before_script: echo skipping gfmxr on $TRAVIS_GO_VERSION
@@ -25,8 +28,12 @@ matrix:
 
 before_script:
 - go get github.com/urfave/gfmxr/...
+- $pip_install flake8
 
 script:
+- flake8 runtests cli-v1-to-v2
 - ./runtests vet
 - ./runtests test
 - ./runtests gfmxr
+- ./cli-v1-to-v2 --selftest
+- ./runtests migrations

CHANGELOG.md

@@ -15,8 +15,8 @@
 
 ### Removed
 - the ability to specify `&StringSlice{...string}` or `&IntSlice{...int}`.
-  To migrate to the new API, you may choose to run [this helper
+  To migrate to the new API, you may choose to run [the migrator
-  (python) script](./cli-migrate-slice-types).
+  (python) script](./cli-v1-to-v2).
 - The optimistic reordering of arguments and flags introduced by
   https://github.com/codegangsta/cli/pull/36. This behavior only worked when
   all arguments appeared before all flags, but caused [weird issues with boolean

README.md

@@ -73,6 +73,9 @@ import (
 ...
 ```
 
+**NOTE**: There is a [migrator (python) script](./cli-v1-to-v2) available to aid
+with the transition from the v1 to v2 API.
+
 ### Pinning to the `v1` branch
 
 Similarly to the section above describing use of the `v2` branch, if one wants

appveyor.yml

@@ -23,3 +23,5 @@ build_script:
 - python runtests vet
 - python runtests test
 - python runtests gfmxr
+- python cli-v1-to-v2 --selftest
+- python runtests migrations

cli-migrate-slice-types

@@ -1,75 +0,0 @@
-#!/usr/bin/env python
-from __future__ import print_function, unicode_literals
-
-import argparse
-import os
-import re
-import sys
-
-
-DESCRIPTION = """\
-Migrate arbitrary `.go` sources from the pre-v2.0.0 API for StringSlice and
-IntSlice types, optionally writing the changes back to file.
-"""
-SLICE_TYPE_RE = re.compile(
-    '&cli\\.(?P<type>IntSlice|StringSlice){(?P<args>[^}]*)}',
-    flags=re.DOTALL
-)
-
-
-def main(sysargs=sys.argv[:]):
-    parser = argparse.ArgumentParser(
-        description=DESCRIPTION,
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    parser.add_argument('basedir', nargs='?', metavar='BASEDIR',
-        type=os.path.abspath, default=os.getcwd())
-    parser.add_argument('-w', '--write', help='write changes back to file',
-        action='store_true', default=False)
-
-    args = parser.parse_args(sysargs[1:])
-
-    for filepath in _find_candidate_files(args.basedir):
-        updated_source = _update_source(filepath)
-        if args.write:
-            print('Updating {!r}'.format(filepath))
-
-            with open(filepath, 'w') as outfile:
-                outfile.write(updated_source)
-        else:
-            print('// -> Updated {!r}'.format(filepath))
-            print(updated_source)
-
-    return 0
-
-
-def _update_source(filepath):
-    with open(filepath) as infile:
-        source = infile.read()
-        return SLICE_TYPE_RE.sub(_slice_type_repl, source)
-
-
-def _slice_type_repl(match):
-    return 'cli.New{}({})'.format(
-        match.groupdict()['type'], match.groupdict()['args']
-    )
-
-
-def _find_candidate_files(basedir):
-    for curdir, dirs, files in os.walk(basedir):
-        for i, dirname in enumerate(dirs[:]):
-            if dirname.startswith('.'):
-                dirs.pop(i)
-
-        for filename in files:
-            if not filename.endswith('.go'):
-                continue
-
-            filepath = os.path.join(curdir, filename)
-            if not os.access(filepath, os.R_OK | os.W_OK):
-                continue
-
-            yield filepath
-
-
-if __name__ == '__main__':
-    sys.exit(main())

cli-v1-to-v2

@@ -0,0 +1,440 @@
+#!/usr/bin/env python
+from __future__ import print_function, unicode_literals
+
+import argparse
+import io
+import logging
+import os
+import re
+import sys
+
+
+_DESCRIPTION = """\
+Migrate arbitrary `.go` sources (mostly) from the v1 to v2 API.
+"""
+_MIGRATORS = []
+
+
+def main(sysargs=sys.argv[:]):
+    parser = argparse.ArgumentParser(
+        description=_DESCRIPTION,
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument('path', nargs='*',
+                        type=os.path.abspath, default=os.getcwd())
+    parser.add_argument('-w', '--write', help='write changes back to file',
+                        action='store_true', default=False)
+    parser.add_argument('-q', '--quiet', help='quiet down the logging',
+                        action='store_true', default=False)
+    parser.add_argument('-D', '--debug', help='debug up the logging',
+                        action='store_true',
+                        default=(os.environ.get('DEBUG') != ''))
+    parser.add_argument('--selftest', help='run internal tests',
+                        action='store_true', default=False)
+
+    args = parser.parse_args(sysargs[1:])
+
+    if args.selftest:
+        logging.basicConfig(
+            level=logging.WARN,
+            format='selftest: %(message)s'
+        )
+        test_migrators()
+        return 0
+
+    level = logging.FATAL if args.quiet else logging.INFO
+    level = logging.DEBUG if args.debug else level
+
+    logging.basicConfig(level=level, format='%(message)s')
+
+    paths = args.path
+    if len(paths) == 0:
+        paths = ['.']
+
+    for filepath in _find_candidate_files(paths):
+        updated_source = _update_filepath(filepath)
+        if args.write:
+            logging.info('Updating %s', filepath)
+
+            with io.open(filepath, 'w', encoding='utf-8') as outfile:
+                outfile.write(updated_source)
+        else:
+            logging.info('// Updated %s:', filepath)
+            print(updated_source)
+
+    return 0
+
+
+def _find_candidate_files(paths):
+    for path in paths:
+        if not os.path.isdir(path):
+            yield path
+            continue
+
+        for curdir, dirs, files in os.walk(path):
+            for i, dirname in enumerate(dirs[:]):
+                if dirname.startswith('.'):
+                    dirs.pop(i)
+
+            for filename in files:
+                if not filename.decode('utf-8').endswith('.go'):
+                    continue
+
+                filepath = os.path.join(curdir, filename)
+                if not os.access(filepath, os.R_OK | os.W_OK):
+                    continue
+
+                yield filepath
+
+
+def _update_filepath(filepath):
+    with io.open(filepath, encoding='utf-8') as infile:
+        return _update_source(infile.read())
+
+
+def _update_source(source):
+    for migrator, func in _MIGRATORS:
+        logging.debug('Running %s migrator', migrator)
+        source = func(source)
+    return source
+
+
+def _subfmt(pattern, replfmt, source, flags=re.UNICODE):
+    def repl(match):
+        return replfmt.format(**match.groupdict())
+    return re.sub(pattern, repl, source, flags=flags)
+
+
+def _migrator(func):
+    _MIGRATORS.append((func.__name__.strip('_'), func))
+    return func
+
+
+@_migrator
+def _slice_pointer_types(source):
+    return _subfmt(
+        '(?P<prefix>\\[\\])cli\\.(?P<type>Command|Author){',
+        '{prefix}*cli.{type}{{', source
+    )
+
+
+@_migrator
+def _pointer_type_literal(source):
+    return _subfmt(
+        '(?P<prefix>\\s+)cli\\.(?P<type>Command|Author){',
+        '{prefix}&cli.{type}{{', source
+    )
+
+
+@_migrator
+def _slice_types(source):
+    return _subfmt(
+        '&cli\\.(?P<type>IntSlice|StringSlice){(?P<args>[^}]*)}',
+        'cli.New{type}({args})', source, flags=re.DOTALL | re.UNICODE
+    )
+
+
+@_migrator
+def _flag_literals(source):
+    return _subfmt(
+        '(?P<prefix>\\s+)cli\\.(?P<type>\\w+)Flag{',
+        '{prefix}&cli.{type}Flag{{', source
+    )
+
+
+@_migrator
+def _v1_imports(source):
+    return re.sub(
+        '"(?:github\\.com|gopkg\\.in)/(?:codegangsta|urfave)/cli(?:\\.v1|)"',
+        '"gopkg.in/urfave/cli.v2"', source, flags=re.UNICODE
+    )
+
+
+@_migrator
+def _new_exit_error(source):
+    return re.sub('cli\\.NewExitError', 'cli.Exit', source, flags=re.UNICODE)
+
+
+@_migrator
+def _bool_t_flag(source):
+    return _subfmt(
+        'cli\\.BoolTFlag{(?P<args>[^}]*)}',
+        'cli.BoolFlag{{Value: true,{args}}}',
+        source, flags=re.DOTALL | re.UNICODE
+    )
+
+
+@_migrator
+def _context_args_len(source):
+    return _subfmt(
+        'len\\((?P<prefix>\\S+)\\.Args\\(\\)\\)',
+        '{prefix}.Args().Len()', source
+    )
+
+
+@_migrator
+def _context_args_index(source):
+    return _subfmt(
+        '\\.Args\\(\\)\\[(?P<index>\\d+)\\]',
+        '.Args().Get({index})', source
+    )
+
+
+@_migrator
+def _envvar_string(source):
+    return re.sub(
+        'EnvVar:(?P<ws>\\s+)"(?P<string>[^"]+)"',
+        _envvar_string_repl, source, flags=re.UNICODE
+    )
+
+
+def _envvar_string_repl(match):
+    return 'EnvVars:{ws}[]string{{{value}}}'.format(
+        value=', '.join([
+            '"{}"'.format(s) for s in
+            re.split(
+                '\\s*,\\s*', match.groupdict()['string'],
+                flags=re.UNICODE
+            )
+        ]),
+        **match.groupdict()
+    )
+
+
+@_migrator
+def _flag_name_stringly(source):
+    return re.sub(
+        '(?P<prefix>\\s+)Name:(?P<ws>\\s+)"(?P<string>[^"]+)"',
+        _flag_name_stringly_repl, source, flags=re.UNICODE
+    )
+
+
+def _flag_name_stringly_repl(match):
+    revars = dict(match.groupdict())
+
+    string = revars['string']
+    parts = list(
+        reversed(
+            sorted(
+                filter(lambda s: len(s.strip()) > 0, [
+                    part.strip() for part in string.split(',')
+                ]), key=len
+            )
+        )
+    )
+
+    if len(parts) == 1:
+        return '{prefix}Name:{ws}"{string}"'.format(**revars)
+
+    return (
+        '{prefix}Name:{ws}"{name}", Aliases: []string{{{aliases}}}'
+    ).format(
+        name=parts[0],
+        aliases=', '.join(['"{}"'.format(s) for s in parts[1:]]),
+        **revars
+    )
+
+
+@_migrator
+def _commands_opaque_type(source):
+    return re.sub('cli\\.Commands', '[]*cli.Command', source, flags=re.UNICODE)
+
+
+@_migrator
+def _flag_names(source):
+    return re.sub('\\.GetName\\(\\)', '.Names()[0]', source, flags=re.UNICODE)
+
+
+@_migrator
+def _app_categories(source):
+    source = _subfmt(
+        '(?P<prefix>range\\s+\\S+)\\.App\\.Categories\\(\\)',
+        '{prefix}.App.Categories.Categories()', source
+    )
+
+    return re.sub(
+        '\\.App\\.Categories\\(\\)', '.App.Categories',
+        source, flags=re.UNICODE
+    )
+
+
+@_migrator
+def _command_category_commands(source):
+    # XXX: brittle
+    return _subfmt(
+        '(?P<prefix>\\s+category\\.)Commands(?P<suffix>[^(])',
+        '{prefix}VisibleCommands(){suffix}', source
+    )
+
+
+@_migrator
+def _context_bool_t(source):
+    # XXX: probably brittle
+    return _subfmt(
+        '(?P<prefix>\\S+)(?:Global|)BoolT\\(',
+        '!{prefix}Bool(', source
+    )
+
+
+@_migrator
+def _context_global_methods(source):
+    return _subfmt(
+        '\\.Global(?P<method>'
+        'Bool|Duration|Float64|Generic|Int|IntSlice|String|StringSlice|'
+        'FlagNames|IsSet|Set'
+        ')\\(',
+        '.{method}(', source
+    )
+
+
+@_migrator
+def _context_parent(source):
+    # XXX: brittle
+    return re.sub('\\.Parent\\(\\)', '.Lineage()[1]', source, flags=re.UNICODE)
+
+
+def test_migrators():
+    import difflib
+
+    for i, (source, expected) in enumerate(_MIGRATOR_TESTS):
+        actual = _update_source(source)
+        if expected != actual:
+            udiff = difflib.unified_diff(
+                expected.splitlines(), actual.splitlines(),
+                fromfile='a/source.go', tofile='b/source.go', lineterm=''
+            )
+            for line in udiff:
+                print(line)
+            raise AssertionError('migrated source does not match expected')
+        logging.warn('Test case %d/%d OK', i+1, len(_MIGRATOR_TESTS))
+
+
+_MIGRATOR_TESTS = (
+    ("""
+\t\t\t&cli.StringSlice{"a", "b", "c"},
+""", """
+\t\t\tcli.NewStringSlice("a", "b", "c"),
+"""),
+    ("""
+\t\tcli.IntFlag{
+\t\t\tName:  "yep",
+\t\t\tValue: 3,
+\t\t}
+""", """
+\t\t&cli.IntFlag{
+\t\t\tName:  "yep",
+\t\t\tValue: 3,
+\t\t}
+"""),
+    ("""
+\t\tapp.Commands = []cli.Command{
+\t\t\t{
+\t\t\t\tName: "whatebbs",
+\t\t\t},
+\t\t}
+""", """
+\t\tapp.Commands = []*cli.Command{
+\t\t\t{
+\t\t\t\tName: "whatebbs",
+\t\t\t},
+\t\t}
+"""),
+    ("""
+\t\tapp.Commands = []cli.Command{
+\t\t\tcli.Command{
+\t\t\t\tName: "whatebbs",
+\t\t\t},
+\t\t}
+""", """
+\t\tapp.Commands = []*cli.Command{
+\t\t\t&cli.Command{
+\t\t\t\tName: "whatebbs",
+\t\t\t},
+\t\t}
+"""),
+    ("""
+\t"github.com/codegangsta/cli"
+\t"github.com/urfave/cli"
+\t"gopkg.in/codegangsta/cli"
+\t"gopkg.in/codegangsta/cli.v1"
+\t"gopkg.in/urfave/cli"
+\t"gopkg.in/urfave/cli.v1"
+""", """
+\t"gopkg.in/urfave/cli.v2"
+\t"gopkg.in/urfave/cli.v2"
+\t"gopkg.in/urfave/cli.v2"
+\t"gopkg.in/urfave/cli.v2"
+\t"gopkg.in/urfave/cli.v2"
+\t"gopkg.in/urfave/cli.v2"
+"""),
+    ("""
+\t\t\t\treturn cli.NewExitError("foo whatebber", 9)
+""", """
+\t\t\t\treturn cli.Exit("foo whatebber", 9)
+"""),
+    ("""
+\t\t\tapp.Flags = []cli.Flag{
+\t\t\t\tcli.StringFlag{
+\t\t\t\t\tName: "aha",
+\t\t\t\t},
+\t\t\t\tcli.BoolTFlag{
+\t\t\t\t\tName: "blurp",
+\t\t\t\t},
+\t\t\t}
+""", """
+\t\t\tapp.Flags = []cli.Flag{
+\t\t\t\t&cli.StringFlag{
+\t\t\t\t\tName: "aha",
+\t\t\t\t},
+\t\t\t\t&cli.BoolFlag{Value: true,
+\t\t\t\t\tName: "blurp",
+\t\t\t\t},
+\t\t\t}
+"""),
+    ("""
+\t\t\tAction = func(c *cli.Context) error {
+\t\t\t\tif c.Args()[4] == "meep" {
+\t\t\t\t\treturn nil
+\t\t\t\t}
+\t\t\t\treturn errors.New("mope")
+\t\t\t}
+""", """
+\t\t\tAction = func(c *cli.Context) error {
+\t\t\t\tif c.Args().Get(4) == "meep" {
+\t\t\t\t\treturn nil
+\t\t\t\t}
+\t\t\t\treturn errors.New("mope")
+\t\t\t}
+"""),
+    ("""
+\t\tapp.Flags = []cli.Flag{
+\t\t\tcli.StringFlag{
+\t\t\t\tName:   "toots",
+\t\t\t\tEnvVar: "TOOTS,TOOTERS",
+\t\t\t},
+\t\t}
+""", """
+\t\tapp.Flags = []cli.Flag{
+\t\t\t&cli.StringFlag{
+\t\t\t\tName:   "toots",
+\t\t\t\tEnvVars: []string{"TOOTS", "TOOTERS"},
+\t\t\t},
+\t\t}
+"""),
+    ("""
+\t\tapp.Flags = []cli.Flag{
+\t\t\tcli.StringFlag{
+\t\t\t\tName:   "t, tootles, toots",
+\t\t\t},
+\t\t}
+""", """
+\t\tapp.Flags = []cli.Flag{
+\t\t\t&cli.StringFlag{
+\t\t\t\tName:   "tootles", Aliases: []string{"toots", "t"},
+\t\t\t},
+\t\t}
+""")
+)
+
+
+if __name__ == '__main__':
+    sys.exit(main())

runtests

@@ -2,35 +2,38 @@
 from __future__ import print_function
 
 import argparse
+import glob
 import os
+import shutil
 import sys
 import tempfile
 
 from subprocess import check_call, check_output
 
 
-PACKAGE_NAME = os.environ.get(
+_PACKAGE_NAME = os.environ.get(
     'CLI_PACKAGE_NAME', 'github.com/urfave/cli'
 )
+_TARGETS = {}
 
 
 def main(sysargs=sys.argv[:]):
-    targets = {
-        'vet': _vet,
-        'test': _test,
-        'gfmxr': _gfmxr
-    }
-
     parser = argparse.ArgumentParser()
     parser.add_argument(
-        'target', nargs='?', choices=tuple(targets.keys()), default='test'
+        'target', nargs='?', choices=tuple(_TARGETS.keys()), default='test'
     )
     args = parser.parse_args(sysargs[1:])
 
-    targets[args.target]()
+    _TARGETS[args.target]()
     return 0
 
 
+def _target(func):
+    _TARGETS[func.__name__.strip('_')] = func
+    return func
+
+
+@_target
 def _test():
     if check_output('go version'.split()).split()[2] < 'go1.2':
         _run('go test -v .'.split())
@@ -46,7 +49,7 @@ def _test():
 
         _run('go test -v'.split() + [
             '-coverprofile={}'.format(coverprofile),
-            ('{}/{}'.format(PACKAGE_NAME, subpackage)).rstrip('/')
+            ('{}/{}'.format(_PACKAGE_NAME, subpackage)).rstrip('/')
         ])
 
     combined_name = _combine_coverprofiles(coverprofiles)
@@ -54,14 +57,41 @@ def _test():
     os.remove(combined_name)
 
 
+@_target
 def _gfmxr():
     _run(['gfmxr', '-c', str(_gfmxr_count()), '-s', 'README.md'])
 
 
+@_target
 def _vet():
     _run('go vet ./...'.split())
 
 
+@_target
+def _migrations():
+    migration_script = os.path.abspath(
+        os.environ.get('V1TOV2', './cli-v1-to-v2')
+    )
+    v1_readme_url = os.environ.get(
+        'V1README',
+        'https://raw.githubusercontent.com/urfave/cli/v1/README.md'
+    )
+
+    tmpdir = tempfile.mkdtemp()
+    try:
+        os.chdir(tmpdir)
+        _run('curl -sSL -o README.md {}'.format(v1_readme_url).split())
+        _run('gfmxr extract -o .'.split())
+
+        for gofile in glob.glob('*.go'):
+            for i in (0, 1):
+                _run(['python', migration_script, '-w', gofile])
+                _run('go build -o tmp.out {}'.format(gofile).split())
+    finally:
+        if os.environ.get('NOCLEAN', '') == '':
+            shutil.rmtree(tmpdir, ignore_errors=True)
+
+
 def _run(command):
     print('runtests: {}'.format(' '.join(command)), file=sys.stderr)
     check_call(command)