From da7819296e4d195bc3484f3bea545c8a11323ece Mon Sep 17 00:00:00 2001 From: Iryna Cherniavska Date: Sun, 21 Dec 2014 09:15:27 -0800 Subject: [PATCH 1/6] one approach to fix completions with function call and lparen --- pgcli/packages/sqlcompletion.py | 24 ++++++++++++++++-------- pgcli/pgcompleter.py | 14 ++++++++------ 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/pgcli/packages/sqlcompletion.py b/pgcli/packages/sqlcompletion.py index b1fc3a808..149a703b9 100644 --- a/pgcli/packages/sqlcompletion.py +++ b/pgcli/packages/sqlcompletion.py @@ -7,7 +7,9 @@ def suggest_type(full_text, text_before_cursor): """Takes the full_text that is typed so far and also the text before the cursor to suggest completion type and scope. - Returns a tuple with a type of entity ('table', 'column' etc) and a scope. + Returns a tuple with a type of entity ('table', 'column' etc), a scope + and a flag to tell the caller to ignore the word at cursor and return all + matches in current scope. A scope for a column category will be a list of tables. """ @@ -31,19 +33,25 @@ def suggest_type(full_text, text_before_cursor): last_token = last_token.value if last_token else '' def is_function_word(word): + """ + Checks iw we are at function call, such as MAX(, MIN(, AVG(, etc. + In this case, we want to return all columns. + :param word: word at cursor, for example MAX( + :return: true or false + """ return word and len(word) > 1 and word[-1] == '(' if is_function_word(word_before_cursor): - return ('columns', extract_tables(full_text)) + return ('columns', extract_tables(full_text), True) elif last_token.lower() in ('set', 'by', 'distinct'): - return ('columns', extract_tables(full_text)) + return ('columns', extract_tables(full_text), False) elif last_token.lower() in ('select', 'where', 'having'): - return ('columns-and-functions', extract_tables(full_text)) + return ('columns-and-functions', extract_tables(full_text), False) elif last_token.lower() in ('from', 'update', 'into', 'describe'): - return ('tables', []) + return ('tables', [], False) elif last_token in ('d',): # \d - return ('tables', []) + return ('tables', [], False) elif last_token.lower() in ('c', 'use'): # \c - return ('databases', []) + return ('databases', [], False) else: - return ('keywords', []) + return ('keywords', [], False) diff --git a/pgcli/pgcompleter.py b/pgcli/pgcompleter.py index 2f0fd6fa8..9b7643bd4 100644 --- a/pgcli/pgcompleter.py +++ b/pgcli/pgcompleter.py @@ -83,24 +83,26 @@ def get_completions(self, document, complete_event): if not self.smart_completion: return self.find_matches(word_before_cursor, self.all_completions) - category, scope = suggest_type(document.text, + category, scope, match_all = suggest_type(document.text, document.text_before_cursor) + word_to_match = '' if match_all else word_before_cursor + if category == 'columns': scoped_cols = [] for table in scope: scoped_cols.extend(self.columns[table]) - return self.find_matches(word_before_cursor, scoped_cols) + return self.find_matches(word_to_match, scoped_cols) elif category == 'columns-and-functions': scoped_cols = [] for table in scope: scoped_cols.extend(self.columns[table]) - return self.find_matches(word_before_cursor, scoped_cols + + return self.find_matches(word_to_match, scoped_cols + self.functions) elif category == 'tables': - return self.find_matches(word_before_cursor, self.tables) + return self.find_matches(word_to_match, self.tables) elif category == 'databases': - return self.find_matches(word_before_cursor, self.databases) + return self.find_matches(word_to_match, self.databases) elif category == 'keywords': - return self.find_matches(word_before_cursor, self.keywords + + return self.find_matches(word_to_match, self.keywords + self.special_commands) From 1bad42f5f33d89ba5a291754a4b6fc39ce14e598 Mon Sep 17 00:00:00 2001 From: Iryna Cherniavska Date: Tue, 23 Dec 2014 13:43:53 -0800 Subject: [PATCH 2/6] Revert "one approach to fix completions with function call and lparen" This reverts commit da7819296e4d195bc3484f3bea545c8a11323ece. --- pgcli/packages/sqlcompletion.py | 24 ++++++++---------------- pgcli/pgcompleter.py | 14 ++++++-------- 2 files changed, 14 insertions(+), 24 deletions(-) diff --git a/pgcli/packages/sqlcompletion.py b/pgcli/packages/sqlcompletion.py index f4c46a3f2..c6029ae0a 100644 --- a/pgcli/packages/sqlcompletion.py +++ b/pgcli/packages/sqlcompletion.py @@ -7,9 +7,7 @@ def suggest_type(full_text, text_before_cursor): """Takes the full_text that is typed so far and also the text before the cursor to suggest completion type and scope. - Returns a tuple with a type of entity ('table', 'column' etc), a scope - and a flag to tell the caller to ignore the word at cursor and return all - matches in current scope. + Returns a tuple with a type of entity ('table', 'column' etc) and a scope. A scope for a column category will be a list of tables. """ @@ -33,25 +31,19 @@ def suggest_type(full_text, text_before_cursor): last_token = last_token.value if last_token else '' def is_function_word(word): - """ - Checks iw we are at function call, such as MAX(, MIN(, AVG(, etc. - In this case, we want to return all columns. - :param word: word at cursor, for example MAX( - :return: true or false - """ return word and len(word) > 1 and word[-1] == '(' if is_function_word(word_before_cursor): - return ('columns', extract_tables(full_text), True) + return ('columns', extract_tables(full_text)) elif last_token.lower() in ('set', 'by', 'distinct'): - return ('columns', extract_tables(full_text), False) + return ('columns', extract_tables(full_text)) elif last_token.lower() in ('select', 'where', 'having'): - return ('columns-and-functions', extract_tables(full_text), False) + return ('columns-and-functions', extract_tables(full_text)) elif last_token.lower() in ('from', 'update', 'into', 'describe'): - return ('tables', [], False) + return ('tables', []) elif last_token in ('d',): # \d - return ('tables', [], False) + return ('tables', []) elif last_token.lower() in ('c', 'use'): # \c - return ('databases', [], False) + return ('databases', []) else: - return ('keywords', [], False) + return ('keywords', []) diff --git a/pgcli/pgcompleter.py b/pgcli/pgcompleter.py index a85551e7d..c5c21facc 100644 --- a/pgcli/pgcompleter.py +++ b/pgcli/pgcompleter.py @@ -83,26 +83,24 @@ def get_completions(self, document, complete_event): if not self.smart_completion: return self.find_matches(word_before_cursor, self.all_completions) - category, scope, match_all = suggest_type(document.text, + category, scope = suggest_type(document.text, document.text_before_cursor) - word_to_match = '' if match_all else word_before_cursor - if category == 'columns': scoped_cols = [] for table in scope: scoped_cols.extend(self.columns[table]) - return self.find_matches(word_to_match, scoped_cols) + return self.find_matches(word_before_cursor, scoped_cols) elif category == 'columns-and-functions': scoped_cols = [] for table in scope: scoped_cols.extend(self.columns[table]) - return self.find_matches(word_to_match, scoped_cols + + return self.find_matches(word_before_cursor, scoped_cols + self.functions) elif category == 'tables': - return self.find_matches(word_to_match, self.tables) + return self.find_matches(word_before_cursor, self.tables) elif category == 'databases': - return self.find_matches(word_to_match, self.databases) + return self.find_matches(word_before_cursor, self.databases) elif category == 'keywords': - return self.find_matches(word_to_match, self.keywords + + return self.find_matches(word_before_cursor, self.keywords + self.special_commands) From df898af993c1a7271b9f5efb3842e5179303f8a8 Mon Sep 17 00:00:00 2001 From: Iryna Cherniavska Date: Tue, 23 Dec 2014 15:01:55 -0800 Subject: [PATCH 3/6] added some smart completion tests --- tests/test_smart_completion.py | 61 ++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/tests/test_smart_completion.py b/tests/test_smart_completion.py index e69de29bb..eec72abd6 100644 --- a/tests/test_smart_completion.py +++ b/tests/test_smart_completion.py @@ -0,0 +1,61 @@ +import pytest +from prompt_toolkit.completion import Completion +from prompt_toolkit.document import Document + +tables = { + 'users': ['id', 'email', 'first_name', 'last_name'], + 'orders': ['id', 'ordered_date', 'status'] +} + +@pytest.fixture +def completer(): + + import pgcli.pgcompleter as pgcompleter + comp = pgcompleter.PGCompleter(smart_completion=True) + comp.extend_table_names(tables.keys()) + for t in tables: + comp.extend_column_names(t, tables[t]) + return comp + +@pytest.fixture +def complete_event(): + from mock import Mock + return Mock() + +def test_empty_string_completion(completer, complete_event): + text = '' + position = 0 + result = set( + completer.get_completions( + Document(text=text, cursor_position=position), + complete_event)) + assert set(map(Completion, completer.keywords)) == result + +def test_select_keyword_completion(completer, complete_event): + text = 'SEL' + position = len('SEL') + result = completer.get_completions( + Document(text=text, cursor_position=position), + complete_event) + assert set(result) == set([Completion(text='SELECT', start_position=-3)]) + +def test_function_name_completion(completer, complete_event): + text = 'SELECT MA' + position = len('SELECT MA') + result = completer.get_completions( + Document(text=text, cursor_position=position), + complete_event) + assert set(result) == set([Completion(text='MAX', start_position=-2)]) + +def test_suggested_column_names_in_function(completer, complete_event): + text = 'SELECT MAX( from users' + position = len('SELECT MAX(') + result = completer.get_completions( + Document(text=text, cursor_position=position), + complete_event) + assert set(result) == set([ + Completion(text='*', start_position=0), + Completion(text='id', start_position=0), + Completion(text='email', start_position=0), + Completion(text='first_name', start_position=0), + Completion(text='last_name', start_position=0)]) From d16dea4b11fb48103bf3f7fcc79c3a59a6ba524b Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Sat, 20 Dec 2014 23:53:47 -0800 Subject: [PATCH 4/6] TODO cleanup. --- TODO | 10 ---------- pgcli/packages/sqlcompletion.py | 24 ++++++++++++++++-------- pgcli/pgcompleter.py | 14 ++++++++------ 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/TODO b/TODO index 674237ebd..828728e67 100644 --- a/TODO +++ b/TODO @@ -27,13 +27,3 @@ * [ ] Set multi-line via config file. * [ ] New Feature List - Write the current version to config file. At launch if the version has changed, display the changelog between the two versions. * [o] Separate the column completions to be table specific. (SELECT, INSERT, UPDATE) -* [X] Default host should not be set to localhost. Since it causes problems in IPv6 machines. Need to research this one further. -* [X] Write a doc about how to run it in develop mode. (pip install -e .) -* [X] Use a pager to display the output. (Check Click's document). -* [X] Column completion for simple SELECT. -* [X] Column completion for simple INSERT. -* [X] Column completion for simple UPDATE. -* [X] Enable multi-line mode via a keybinding. -* [X] Pressing enter should just pop another prompt. -* [X] Implement \?. -* [X] Automate the release procedure. diff --git a/pgcli/packages/sqlcompletion.py b/pgcli/packages/sqlcompletion.py index b1fc3a808..149a703b9 100644 --- a/pgcli/packages/sqlcompletion.py +++ b/pgcli/packages/sqlcompletion.py @@ -7,7 +7,9 @@ def suggest_type(full_text, text_before_cursor): """Takes the full_text that is typed so far and also the text before the cursor to suggest completion type and scope. - Returns a tuple with a type of entity ('table', 'column' etc) and a scope. + Returns a tuple with a type of entity ('table', 'column' etc), a scope + and a flag to tell the caller to ignore the word at cursor and return all + matches in current scope. A scope for a column category will be a list of tables. """ @@ -31,19 +33,25 @@ def suggest_type(full_text, text_before_cursor): last_token = last_token.value if last_token else '' def is_function_word(word): + """ + Checks iw we are at function call, such as MAX(, MIN(, AVG(, etc. + In this case, we want to return all columns. + :param word: word at cursor, for example MAX( + :return: true or false + """ return word and len(word) > 1 and word[-1] == '(' if is_function_word(word_before_cursor): - return ('columns', extract_tables(full_text)) + return ('columns', extract_tables(full_text), True) elif last_token.lower() in ('set', 'by', 'distinct'): - return ('columns', extract_tables(full_text)) + return ('columns', extract_tables(full_text), False) elif last_token.lower() in ('select', 'where', 'having'): - return ('columns-and-functions', extract_tables(full_text)) + return ('columns-and-functions', extract_tables(full_text), False) elif last_token.lower() in ('from', 'update', 'into', 'describe'): - return ('tables', []) + return ('tables', [], False) elif last_token in ('d',): # \d - return ('tables', []) + return ('tables', [], False) elif last_token.lower() in ('c', 'use'): # \c - return ('databases', []) + return ('databases', [], False) else: - return ('keywords', []) + return ('keywords', [], False) diff --git a/pgcli/pgcompleter.py b/pgcli/pgcompleter.py index 2f0fd6fa8..9b7643bd4 100644 --- a/pgcli/pgcompleter.py +++ b/pgcli/pgcompleter.py @@ -83,24 +83,26 @@ def get_completions(self, document, complete_event): if not self.smart_completion: return self.find_matches(word_before_cursor, self.all_completions) - category, scope = suggest_type(document.text, + category, scope, match_all = suggest_type(document.text, document.text_before_cursor) + word_to_match = '' if match_all else word_before_cursor + if category == 'columns': scoped_cols = [] for table in scope: scoped_cols.extend(self.columns[table]) - return self.find_matches(word_before_cursor, scoped_cols) + return self.find_matches(word_to_match, scoped_cols) elif category == 'columns-and-functions': scoped_cols = [] for table in scope: scoped_cols.extend(self.columns[table]) - return self.find_matches(word_before_cursor, scoped_cols + + return self.find_matches(word_to_match, scoped_cols + self.functions) elif category == 'tables': - return self.find_matches(word_before_cursor, self.tables) + return self.find_matches(word_to_match, self.tables) elif category == 'databases': - return self.find_matches(word_before_cursor, self.databases) + return self.find_matches(word_to_match, self.databases) elif category == 'keywords': - return self.find_matches(word_before_cursor, self.keywords + + return self.find_matches(word_to_match, self.keywords + self.special_commands) From 0f07cb874e87a64e708f74e6282d00f1e5d09773 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Mon, 22 Dec 2014 22:43:43 -0800 Subject: [PATCH 5/6] Strip only a limited set of punctuations before matching. --- TODO | 2 +- pgcli/packages/parseutils.py | 16 ++++++++++------ pgcli/packages/sqlcompletion.py | 26 +++++++++----------------- pgcli/pgcompleter.py | 16 +++++++--------- 4 files changed, 27 insertions(+), 33 deletions(-) diff --git a/TODO b/TODO index 828728e67..eabcfd0bd 100644 --- a/TODO +++ b/TODO @@ -2,7 +2,6 @@ * [ ] Fix: SELECT id, FROM django_migrations; - Auto-completion for the second column name is broken. Find the last keyword and use it for completion. * [ ] Skip the password prompt by default. It should only be presented if -W option is provided. * [ ] Bottom status bar is cut-off in half pane. Figure out how to fix that. -* [ ] Fix: Autocompletion won't go away after semi-colons. This an artifact of stripping special chars in the partially typed words. Need to selectively remove parens. * [ ] Column completion for nested sql. * [ ] Add JOIN to the list of keywords and provide proper autocompletion for it. * [ ] Improve the smart completion for Insert statement. (Needs table specific columns) @@ -27,3 +26,4 @@ * [ ] Set multi-line via config file. * [ ] New Feature List - Write the current version to config file. At launch if the version has changed, display the changelog between the two versions. * [o] Separate the column completions to be table specific. (SELECT, INSERT, UPDATE) +* [X] Fix: Autocompletion won't go away after semi-colons. This an artifact of stripping special chars in the partially typed words. Need to selectively remove parens. diff --git a/pgcli/packages/parseutils.py b/pgcli/packages/parseutils.py index a58e0225f..f4a5f3339 100644 --- a/pgcli/packages/parseutils.py +++ b/pgcli/packages/parseutils.py @@ -3,12 +3,16 @@ from sqlparse.sql import IdentifierList, Identifier from sqlparse.tokens import Keyword, DML -# This matches only alphanumerics and underscores. -_LAST_WORD_RE = re.compile(r'(\w+)$') -# This matches everything except a space. -_LAST_WORD_SPL_RE = re.compile(r'([^\s]+)$') +cleanup_regex = { + # This matches only alphanumerics and underscores. + 'alphanum_underscore': re.compile(r'(\w+)$'), + # This matches everything except spaces, parens and comma. + 'most_punctuations': re.compile(r'([^(),\s]+)$'), + # This matches everything except a space. + 'all_punctuations': re.compile('([^\s]+)$'), + } -def last_word(text, include_special_chars=False): +def last_word(text, include='alphanum_underscore'): """ Find the last word in a sentence. @@ -44,7 +48,7 @@ def last_word(text, include_special_chars=False): if text[-1].isspace(): return '' else: - regex = _LAST_WORD_SPL_RE if include_special_chars else _LAST_WORD_RE + regex = cleanup_regex[include] matches = regex.search(text) if matches: return matches.group(0) diff --git a/pgcli/packages/sqlcompletion.py b/pgcli/packages/sqlcompletion.py index 149a703b9..c6029ae0a 100644 --- a/pgcli/packages/sqlcompletion.py +++ b/pgcli/packages/sqlcompletion.py @@ -7,14 +7,12 @@ def suggest_type(full_text, text_before_cursor): """Takes the full_text that is typed so far and also the text before the cursor to suggest completion type and scope. - Returns a tuple with a type of entity ('table', 'column' etc), a scope - and a flag to tell the caller to ignore the word at cursor and return all - matches in current scope. + Returns a tuple with a type of entity ('table', 'column' etc) and a scope. A scope for a column category will be a list of tables. """ word_before_cursor = last_word(text_before_cursor, - include_special_chars=True) + include='all_punctuations') # If we've partially typed a word then word_before_cursor won't be an empty # string. In that case we want to remove the partially typed string before @@ -33,25 +31,19 @@ def suggest_type(full_text, text_before_cursor): last_token = last_token.value if last_token else '' def is_function_word(word): - """ - Checks iw we are at function call, such as MAX(, MIN(, AVG(, etc. - In this case, we want to return all columns. - :param word: word at cursor, for example MAX( - :return: true or false - """ return word and len(word) > 1 and word[-1] == '(' if is_function_word(word_before_cursor): - return ('columns', extract_tables(full_text), True) + return ('columns', extract_tables(full_text)) elif last_token.lower() in ('set', 'by', 'distinct'): - return ('columns', extract_tables(full_text), False) + return ('columns', extract_tables(full_text)) elif last_token.lower() in ('select', 'where', 'having'): - return ('columns-and-functions', extract_tables(full_text), False) + return ('columns-and-functions', extract_tables(full_text)) elif last_token.lower() in ('from', 'update', 'into', 'describe'): - return ('tables', [], False) + return ('tables', []) elif last_token in ('d',): # \d - return ('tables', [], False) + return ('tables', []) elif last_token.lower() in ('c', 'use'): # \c - return ('databases', [], False) + return ('databases', []) else: - return ('keywords', [], False) + return ('keywords', []) diff --git a/pgcli/pgcompleter.py b/pgcli/pgcompleter.py index 9b7643bd4..c5c21facc 100644 --- a/pgcli/pgcompleter.py +++ b/pgcli/pgcompleter.py @@ -69,7 +69,7 @@ def reset_completions(self): @staticmethod def find_matches(text, collection): - text = last_word(text) + text = last_word(text, include='most_punctuations') for item in collection: if item.startswith(text) or item.startswith(text.upper()): yield Completion(item, -len(text)) @@ -83,26 +83,24 @@ def get_completions(self, document, complete_event): if not self.smart_completion: return self.find_matches(word_before_cursor, self.all_completions) - category, scope, match_all = suggest_type(document.text, + category, scope = suggest_type(document.text, document.text_before_cursor) - word_to_match = '' if match_all else word_before_cursor - if category == 'columns': scoped_cols = [] for table in scope: scoped_cols.extend(self.columns[table]) - return self.find_matches(word_to_match, scoped_cols) + return self.find_matches(word_before_cursor, scoped_cols) elif category == 'columns-and-functions': scoped_cols = [] for table in scope: scoped_cols.extend(self.columns[table]) - return self.find_matches(word_to_match, scoped_cols + + return self.find_matches(word_before_cursor, scoped_cols + self.functions) elif category == 'tables': - return self.find_matches(word_to_match, self.tables) + return self.find_matches(word_before_cursor, self.tables) elif category == 'databases': - return self.find_matches(word_to_match, self.databases) + return self.find_matches(word_before_cursor, self.databases) elif category == 'keywords': - return self.find_matches(word_to_match, self.keywords + + return self.find_matches(word_before_cursor, self.keywords + self.special_commands) From c08e52b9569401def9216dcb8891a393a281ddc0 Mon Sep 17 00:00:00 2001 From: Iryna Cherniavska Date: Tue, 23 Dec 2014 15:01:55 -0800 Subject: [PATCH 6/6] added some smart completion tests --- tests/test_smart_completion.py | 61 ++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/tests/test_smart_completion.py b/tests/test_smart_completion.py index e69de29bb..eec72abd6 100644 --- a/tests/test_smart_completion.py +++ b/tests/test_smart_completion.py @@ -0,0 +1,61 @@ +import pytest +from prompt_toolkit.completion import Completion +from prompt_toolkit.document import Document + +tables = { + 'users': ['id', 'email', 'first_name', 'last_name'], + 'orders': ['id', 'ordered_date', 'status'] +} + +@pytest.fixture +def completer(): + + import pgcli.pgcompleter as pgcompleter + comp = pgcompleter.PGCompleter(smart_completion=True) + comp.extend_table_names(tables.keys()) + for t in tables: + comp.extend_column_names(t, tables[t]) + return comp + +@pytest.fixture +def complete_event(): + from mock import Mock + return Mock() + +def test_empty_string_completion(completer, complete_event): + text = '' + position = 0 + result = set( + completer.get_completions( + Document(text=text, cursor_position=position), + complete_event)) + assert set(map(Completion, completer.keywords)) == result + +def test_select_keyword_completion(completer, complete_event): + text = 'SEL' + position = len('SEL') + result = completer.get_completions( + Document(text=text, cursor_position=position), + complete_event) + assert set(result) == set([Completion(text='SELECT', start_position=-3)]) + +def test_function_name_completion(completer, complete_event): + text = 'SELECT MA' + position = len('SELECT MA') + result = completer.get_completions( + Document(text=text, cursor_position=position), + complete_event) + assert set(result) == set([Completion(text='MAX', start_position=-2)]) + +def test_suggested_column_names_in_function(completer, complete_event): + text = 'SELECT MAX( from users' + position = len('SELECT MAX(') + result = completer.get_completions( + Document(text=text, cursor_position=position), + complete_event) + assert set(result) == set([ + Completion(text='*', start_position=0), + Completion(text='id', start_position=0), + Completion(text='email', start_position=0), + Completion(text='first_name', start_position=0), + Completion(text='last_name', start_position=0)])