diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..6bf0a02 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,7 @@ +[run] +branch = True +source = yaql +omit = yaql/tests/* + +[report] +ignore-errors = True diff --git a/.gitignore b/.gitignore index 18e1628..1399c98 100644 --- a/.gitignore +++ b/.gitignore @@ -1,16 +1,51 @@ -#IntelJ Idea -.idea/ +*.py[cod] -#virtualenv -.venv/ +# C extensions +*.so -#Build results -build/ -dist/ -*.egg-info/ +# Packages +*.egg +*.egg-info +dist +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 -#Python -*.pyc +# Installer logs +pip-log.txt -#Generated grammar -yaql/parser_table.py +# Unit test / coverage reports +.coverage +.tox +nosetests.xml +.testrepository + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Complexity +output/*.html +output/*/index.html + +# Sphinx +doc/build + +# pbr generates these +AUTHORS +ChangeLog + +# Editors +*~ +.*.swp \ No newline at end of file diff --git a/.gitreview b/.gitreview new file mode 100644 index 0000000..1824f0b --- /dev/null +++ b/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=review.openstack.org +port=29418 +project=stackforge/yaql.git \ No newline at end of file diff --git a/.mailmap b/.mailmap new file mode 100644 index 0000000..cc92f17 --- /dev/null +++ b/.mailmap @@ -0,0 +1,3 @@ +# Format is: +# +# \ No newline at end of file diff --git a/.testr.conf b/.testr.conf new file mode 100644 index 0000000..6d83b3c --- /dev/null +++ b/.testr.conf @@ -0,0 +1,7 @@ +[DEFAULT] +test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ + OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ + OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \ + ${PYTHON:-python} -m subunit.run discover -t ./ . $LISTOPT $IDOPTION +test_id_option=--load-list $IDFILE +test_list_option=--list diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 0000000..ba86be2 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,17 @@ +If you would like to contribute to the development of OpenStack, +you must follow the steps in the "If you're a developer, start here" +section of this page: + + http://wiki.openstack.org/HowToContribute + +Once those steps have been completed, changes to OpenStack +should be submitted for review via the Gerrit tool, following +the workflow documented at: + + http://wiki.openstack.org/GerritWorkflow + +Pull requests submitted through GitHub will be ignored. + +Bugs should be filed on Launchpad, not GitHub: + + https://bugs.launchpad.net/yaql \ No newline at end of file diff --git a/HACKING.rst b/HACKING.rst new file mode 100644 index 0000000..1b5669f --- /dev/null +++ b/HACKING.rst @@ -0,0 +1,4 @@ +yaql Style Commandments +=============================================== + +Read the OpenStack Style Commandments http://docs.openstack.org/developer/hacking/ \ No newline at end of file diff --git a/LICENSE b/LICENSE index 68c771a..67db858 100644 --- a/LICENSE +++ b/LICENSE @@ -173,4 +173,3 @@ defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. - diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..90f8a7a --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +include AUTHORS +include ChangeLog +exclude .gitignore +exclude .gitreview + +global-exclude *.pyc \ No newline at end of file diff --git a/README.md b/README.rst similarity index 98% rename from README.md rename to README.rst index 83afa1c..e583254 100644 --- a/README.md +++ b/README.rst @@ -1,5 +1,5 @@ YAQL - Yet Another Query Language -==== +================================= At the beginning of millennium the growing trend towards data formats standardization and application integrability made XML extremely popular. XML became lingua franca of the data. Applications tended to process lots of XML files ranging @@ -98,4 +98,4 @@ parameter - context. Context is a repository of functions and variables that can be used in expressions. So all the functions above are just ordinary Python functions that are registered in Context object. But because they all need to be registered in Context -user can always customize them, add his own model-specific ones and have full control over the expression evaluation. \ No newline at end of file +user can always customize them, add his own model-specific ones and have full control over the expression evaluation. diff --git a/babel.cfg b/babel.cfg new file mode 100644 index 0000000..efceab8 --- /dev/null +++ b/babel.cfg @@ -0,0 +1 @@ +[python: **.py] diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100755 index 0000000..27f7ae0 --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +sys.path.insert(0, os.path.abspath('../..')) +# -- General configuration ---------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [ + 'sphinx.ext.autodoc', + #'sphinx.ext.intersphinx', + 'oslosphinx' +] + +# autodoc generation is a bit aggressive and a nuisance when doing heavy +# text edit cycles. +# execute "export SPHINX_DEBUG=1" in your terminal to disable + +# The suffix of source filenames. +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'yaql' +copyright = u'2013, OpenStack Foundation' + +# If true, '()' will be appended to :func: etc. cross-reference text. +add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +add_module_names = True + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# -- Options for HTML output -------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +# html_theme_path = ["."] +# html_theme = '_theme' +# html_static_path = ['static'] + +# Output file base name for HTML help builder. +htmlhelp_basename = '%sdoc' % project + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass +# [howto/manual]). +latex_documents = [ + ('index', + '%s.tex' % project, + u'%s Documentation' % project, + u'OpenStack Foundation', 'manual'), +] + +# Example configuration for intersphinx: refer to the Python standard library. +#intersphinx_mapping = {'http://docs.python.org/': None} \ No newline at end of file diff --git a/doc/source/contributing.rst b/doc/source/contributing.rst new file mode 100644 index 0000000..ed77c12 --- /dev/null +++ b/doc/source/contributing.rst @@ -0,0 +1,4 @@ +============ +Contributing +============ +.. include:: ../../CONTRIBUTING.rst \ No newline at end of file diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 0000000..8212b33 --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,24 @@ +.. yaql documentation master file, created by + sphinx-quickstart on Tue Jul 9 22:26:36 2013. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to yaql's documentation! +======================================================== + +Contents: + +.. toctree:: + :maxdepth: 2 + + readme + installation + usage + contributing + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/doc/source/installation.rst b/doc/source/installation.rst new file mode 100644 index 0000000..0f0b3b3 --- /dev/null +++ b/doc/source/installation.rst @@ -0,0 +1,12 @@ +============ +Installation +============ + +At the command line:: + + $ pip install yaql + +Or, if you have virtualenvwrapper installed:: + + $ mkvirtualenv yaql + $ pip install yaql \ No newline at end of file diff --git a/doc/source/readme.rst b/doc/source/readme.rst new file mode 100644 index 0000000..38ba804 --- /dev/null +++ b/doc/source/readme.rst @@ -0,0 +1 @@ +.. include:: ../../README.rst \ No newline at end of file diff --git a/doc/source/usage.rst b/doc/source/usage.rst new file mode 100644 index 0000000..6f73a9e --- /dev/null +++ b/doc/source/usage.rst @@ -0,0 +1,7 @@ +======== +Usage +======== + +To use yaql in a project:: + + import yaql \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d369984 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +pbr>=0.6,!=0.7,<1.0 +Babel>=0.9.6 + +ply diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..9a69e8f --- /dev/null +++ b/setup.cfg @@ -0,0 +1,46 @@ +[metadata] +name = yaql +summary = YAQL - Yet Another Query Language +description-file = + README.rst +author = OpenStack +author-email = openstack-dev@lists.openstack.org +home-page = http://www.openstack.org/ +classifier = + Environment :: OpenStack + Intended Audience :: Information Technology + Intended Audience :: System Administrators + License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 2.6 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.3 + +[files] +packages = + yaql + +[build_sphinx] +source-dir = doc/source +build-dir = doc/build +all_files = 1 + +[upload_sphinx] +upload-dir = doc/build/html + +[compile_catalog] +directory = yaql/locale +domain = yaql + +[update_catalog] +domain = yaql +output_dir = yaql/locale +input_file = yaql/locale/yaql.pot + +[extract_messages] +keywords = _ gettext ngettext l_ lazy_gettext +mapping_file = babel.cfg +output_file = yaql/locale/yaql.pot \ No newline at end of file diff --git a/setup.py b/setup.py index a5ac195..70c2b3f 100644 --- a/setup.py +++ b/setup.py @@ -1,32 +1,22 @@ -# Copyright (c) 2013 Mirantis, Inc. +#!/usr/bin/env python +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. -from setuptools import setup, find_packages +# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT +import setuptools -setup(name='yaql', - version='0.3', - description="Yet Another Query Language", - author='Mirantis, Inc.', - author_email='info@mirantis.com', - url='https://github.com/ativelkov/yaql', - install_requires=['ply'], - entry_points={ - 'console_scripts': [ - 'yaql = yaql.cli.run:main', - ] - }, - packages=find_packages(), - package_data={ - 'examples': ['*.json'], - },) +setuptools.setup( + setup_requires=['pbr'], + pbr=True) diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..2a46bd8 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,11 @@ +hacking>=0.5.6,<0.8 + +coverage>=3.6 +discover +fixtures>=0.3.14 +python-subunit +sphinx>=1.1.2 +oslosphinx +testrepository>=0.0.17 +testscenarios>=0.4,<0.5 +testtools>=0.9.32 \ No newline at end of file diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..ced3b37 --- /dev/null +++ b/tox.ini @@ -0,0 +1,44 @@ +[tox] +minversion = 1.6 +envlist = py26,py27,py33,py34,pypy,pep8 +skipsdist = True + +[testenv] +usedevelop = True +install_command = pip install -U {opts} {packages} +setenv = + VIRTUAL_ENV={envdir} +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +commands = python setup.py testr --slowest --testr-args='{posargs}' + +[testenv:pep8] +commands = flake8 + +[testenv:venv] +commands = {posargs} + +[testenv:cover] +commands = python setup.py testr --coverage --testr-args='{posargs}' + +[testenv:docs] +commands = python setup.py build_sphinx + +[flake8] +# H803 skipped on purpose per list discussion. +# E123, E125 skipped as they are invalid PEP-8. +# H404 multi line docstring should start with a summary +## TODO(ruhe) following checks should be fixed +# E501 line too long +# E721 do not compare types, use 'isinstance()' +# F401 something imported but unused +# F403 import something.* +# F841 local variable 'e' is assigned to but never used +# H201 no 'except:' at least use 'except Exception:' +# H202 assertRaises Exception too broad +# H306 imports not in alphabetical order +# H902 Use the 'not in' operator for collection membership evaluation +show-source = True +ignore = E123,E125,E501,E721,F401,F403,F841,H201,H202,H306,H404,H803,H902 +builtins = _ +exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build diff --git a/yaql/cli/__init__.py b/yaql/cli/__init__.py index 207fa15..e69de29 100644 --- a/yaql/cli/__init__.py +++ b/yaql/cli/__init__.py @@ -1,13 +0,0 @@ -# Copyright (c) 2013 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. \ No newline at end of file diff --git a/yaql/cli/run.py b/yaql/cli/run.py index 39dc06f..c5a894b 100755 --- a/yaql/cli/run.py +++ b/yaql/cli/run.py @@ -32,7 +32,7 @@ def main(): decoder = JSONDecoder() data = decoder.decode(json_str) except: - print "Unable to load data from "+options.data + print "Unable to load data from " + options.data return else: data = None diff --git a/yaql/functions/arithmetic.py b/yaql/functions/arithmetic.py index 335dc82..a007bcb 100644 --- a/yaql/functions/arithmetic.py +++ b/yaql/functions/arithmetic.py @@ -19,6 +19,7 @@ from yaql.language.exceptions import YaqlExecutionException def _is_a_number(value): return isinstance(value, (int, long, float, complex)) + @parameter('value', custom_validator=_is_a_number) def unary_minus(value): return -1 * value @@ -34,6 +35,7 @@ def unary_plus(value): def plus(a, b): return a + b + @parameter('a', custom_validator=_is_a_number) @parameter('b', custom_validator=_is_a_number) def minus(a, b): diff --git a/yaql/functions/boolean.py b/yaql/functions/boolean.py index c2e14ec..94338cc 100644 --- a/yaql/functions/boolean.py +++ b/yaql/functions/boolean.py @@ -21,11 +21,13 @@ from yaql.language.exceptions import YaqlExecutionException def _and(a, b): return a and b + @parameter('a', arg_type=types.BooleanType) @parameter('b', arg_type=types.BooleanType) def _or(a, b): return a or b + @parameter('data', arg_type=types.BooleanType) def _not(data): return not data diff --git a/yaql/functions/containers.py b/yaql/functions/containers.py index fd9aad4..884e2a3 100644 --- a/yaql/functions/containers.py +++ b/yaql/functions/containers.py @@ -92,7 +92,10 @@ def append_tuple(arg1, arg2): def build_dict(*tuples): try: - return {key: value for key, value in tuples} + d = {} + for key, value in tuples: + d[key] = value + return d except ValueError as e: raise YaqlExecutionException("Not a valid dictionary", e) @@ -151,6 +154,7 @@ def take_while(self, predicate): else: return + @parameter('self', arg_type=types.GeneratorType) def _list(self): return limit(self) @@ -180,4 +184,3 @@ def add_to_context(context): context.register_function(take_while) context.register_function(_list, 'list') context.register_function(for_each) - diff --git a/yaql/functions/ns.py b/yaql/functions/ns.py index 4d1b136..974dd38 100644 --- a/yaql/functions/ns.py +++ b/yaql/functions/ns.py @@ -59,6 +59,7 @@ def resolve_prop(alias, symbol, context): namespace.validate(symbol) return namespace.name + '.' + symbol + @context_aware @parameter('symbol', function_only=True, lazy=True) def resolve_function(self, alias, symbol, context): diff --git a/yaql/functions/strings.py b/yaql/functions/strings.py index 9bfa781..0a6a29f 100644 --- a/yaql/functions/strings.py +++ b/yaql/functions/strings.py @@ -21,6 +21,7 @@ from yaql.language.engine import parameter def string_concatenation(a, b): return a + b + @parameter('self', arg_type=types.StringTypes, is_self=True) def as_list(self): return list(self) @@ -29,6 +30,7 @@ def as_list(self): def to_string(self): return str(self) + def _to_string_func(data): return to_string(data) diff --git a/yaql/functions/system.py b/yaql/functions/system.py index a6daea6..7c23996 100644 --- a/yaql/functions/system.py +++ b/yaql/functions/system.py @@ -54,6 +54,7 @@ def dict_attribution(self, att_name): def method_call(self, method): return method(sender=self) + @context_aware @parameter('tuple_preds', lazy=True) def _as(self, context, *tuple_preds): diff --git a/yaql/language/__init__.py b/yaql/language/__init__.py index fa47701..e69de29 100644 --- a/yaql/language/__init__.py +++ b/yaql/language/__init__.py @@ -1,13 +0,0 @@ -# Copyright (c) 2014 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. diff --git a/yaql/language/lexer.py b/yaql/language/lexer.py index 2183d25..9c30346 100644 --- a/yaql/language/lexer.py +++ b/yaql/language/lexer.py @@ -39,19 +39,19 @@ unary_prefix = { op_to_level = { 'abc': 0, - '|' : 1, - '^' : 2, - '&' : 3, - '<' : 4, - '>' : 4, - '=' : 5, - '!' : 5, - '+' : 6, - '-' : 6, - '*' : 7, - '/' : 7, - '%' : 7, - '.' : 8 + '|': 1, + '^': 2, + '&': 3, + '<': 4, + '>': 4, + '=': 5, + '!': 5, + '+': 6, + '-': 6, + '*': 7, + '/': 7, + '%': 7, + '.': 8 } ops = { @@ -86,7 +86,7 @@ tokens = [ 'FILTER', 'NOT', 'DOLLAR' -] + list(keywords.values())+list(ops.values()) + list(unary_prefix.values()) +] + list(keywords.values()) + list(ops.values()) + list(unary_prefix.values()) literals = "()]," @@ -162,12 +162,11 @@ def t_CHAR_ORB(t): return t - def get_orb_op_type(first_char, last_char): if first_char.isalpha() or first_char == '_': level = op_to_level['abc'] else: - level = op_to_level.get(first_char, max(op_to_level.values())+1) + level = op_to_level.get(first_char, max(op_to_level.values()) + 1) asc = 'r' if last_char in right_associative else 'l' return ops.get((level, asc)) diff --git a/yaql/language/parser.py b/yaql/language/parser.py index 09bf679..82374d9 100644 --- a/yaql/language/parser.py +++ b/yaql/language/parser.py @@ -140,7 +140,6 @@ def p_binary(p): p[0] = expressions.BinaryOperator(p[2], p[1], p[3]) - def p_unary_prefix(p): """ value : UNARY_TILDE value diff --git a/yaql/tests/test_execution_chains.py b/yaql/tests/test_execution_chains.py index bf90b89..5616087 100644 --- a/yaql/tests/test_execution_chains.py +++ b/yaql/tests/test_execution_chains.py @@ -50,6 +50,7 @@ def override_with_caps(self, context): def _print(self): return "data is: %s" % self + @parameter('self', arg_type=types.StringType) def print_string(self): return "print %s" % self @@ -63,7 +64,6 @@ class TestExecutionChain(YaqlTest): self.context.register_function(_print, 'print') self.context.register_function(override_with_caps, 'caps_on') - def test_chain1(self): expression = 'f1(abc).f2().f3()' self.assertEquals('f3(f2(f1(abc)))', self.eval(expression)) @@ -103,7 +103,5 @@ class TestExecutionChain(YaqlTest): self.assertRaises(YaqlExecutionException, self.eval, wrong_expression) - - if __name__ == '__main__': unittest.main() diff --git a/yaql/tests/test_tuples.py b/yaql/tests/test_tuples.py index b7968d6..2eba222 100644 --- a/yaql/tests/test_tuples.py +++ b/yaql/tests/test_tuples.py @@ -54,6 +54,5 @@ class TestTuples(YaqlTest): self.assertEquals((5, 'a'), self.eval(expression2)) - if __name__ == '__main__': unittest.main()