From 48ea0f0d0bd8bb30bfd1374cbd0d790ade7a764e Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Sat, 14 May 2011 09:01:41 +0000 Subject: [PATCH 01/90] Rename libcloud.resources.lb to libcloud.loadbalancer' git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1102975 13f79535-47bb-0310-9956-ffa450edef68 --- example_lb.py | 6 +++--- libcloud/{resource/lb => loadbalancer}/__init__.py | 0 libcloud/{resource/lb => loadbalancer}/base.py | 0 .../lb => loadbalancer}/drivers/__init__.py | 0 .../lb => loadbalancer}/drivers/gogrid.py | 4 ++-- .../lb => loadbalancer}/drivers/rackspace.py | 4 ++-- .../{resource/lb => loadbalancer}/providers.py | 6 +++--- libcloud/{resource/lb => loadbalancer}/types.py | 0 setup.py | 7 +++---- test/file_fixtures.py | 6 +++--- test/loadbalancer/__init__.py | 14 ++++++++++++++ .../fixtures}/gogrid/ip_list.json | 0 .../fixtures}/gogrid/loadbalancer_add.json | 0 .../fixtures}/gogrid/loadbalancer_edit.json | 0 .../fixtures}/gogrid/loadbalancer_get.json | 0 .../fixtures}/gogrid/loadbalancer_list.json | 0 .../fixtures}/rackspace/v1_slug_loadbalancers.json | 0 .../rackspace/v1_slug_loadbalancers_8290.json | 0 .../v1_slug_loadbalancers_8290_nodes.json | 0 .../v1_slug_loadbalancers_8290_nodes_post.json | 0 .../rackspace/v1_slug_loadbalancers_post.json | 0 test/{resource/lb => loadbalancer}/test_gogrid.py | 8 ++++---- .../lb => loadbalancer}/test_rackspace.py | 8 ++++---- 23 files changed, 38 insertions(+), 25 deletions(-) rename libcloud/{resource/lb => loadbalancer}/__init__.py (100%) rename libcloud/{resource/lb => loadbalancer}/base.py (100%) rename libcloud/{resource/lb => loadbalancer}/drivers/__init__.py (100%) rename libcloud/{resource/lb => loadbalancer}/drivers/gogrid.py (97%) rename libcloud/{resource/lb => loadbalancer}/drivers/rackspace.py (97%) rename libcloud/{resource/lb => loadbalancer}/providers.py (84%) rename libcloud/{resource/lb => loadbalancer}/types.py (100%) create mode 100644 test/loadbalancer/__init__.py rename test/{resource/fixtures/lb => loadbalancer/fixtures}/gogrid/ip_list.json (100%) rename test/{resource/fixtures/lb => loadbalancer/fixtures}/gogrid/loadbalancer_add.json (100%) rename test/{resource/fixtures/lb => loadbalancer/fixtures}/gogrid/loadbalancer_edit.json (100%) rename test/{resource/fixtures/lb => loadbalancer/fixtures}/gogrid/loadbalancer_get.json (100%) rename test/{resource/fixtures/lb => loadbalancer/fixtures}/gogrid/loadbalancer_list.json (100%) rename test/{resource/fixtures/lb => loadbalancer/fixtures}/rackspace/v1_slug_loadbalancers.json (100%) rename test/{resource/fixtures/lb => loadbalancer/fixtures}/rackspace/v1_slug_loadbalancers_8290.json (100%) rename test/{resource/fixtures/lb => loadbalancer/fixtures}/rackspace/v1_slug_loadbalancers_8290_nodes.json (100%) rename test/{resource/fixtures/lb => loadbalancer/fixtures}/rackspace/v1_slug_loadbalancers_8290_nodes_post.json (100%) rename test/{resource/fixtures/lb => loadbalancer/fixtures}/rackspace/v1_slug_loadbalancers_post.json (100%) rename test/{resource/lb => loadbalancer}/test_gogrid.py (94%) rename test/{resource/lb => loadbalancer}/test_rackspace.py (95%) diff --git a/example_lb.py b/example_lb.py index b4cc69f3b1..0fbd186328 100644 --- a/example_lb.py +++ b/example_lb.py @@ -3,9 +3,9 @@ import os import time -from libcloud.resource.lb.base import LB, LBNode -from libcloud.resource.lb.types import Provider, LBState -from libcloud.resource.lb.providers import get_driver +from libcloud.loadbalancer.base import LB, LBNode +from libcloud.loadbalancer.types import Provider, LBState +from libcloud.loadbalancer.providers import get_driver def main(): Rackspace = get_driver(Provider.RACKSPACE) diff --git a/libcloud/resource/lb/__init__.py b/libcloud/loadbalancer/__init__.py similarity index 100% rename from libcloud/resource/lb/__init__.py rename to libcloud/loadbalancer/__init__.py diff --git a/libcloud/resource/lb/base.py b/libcloud/loadbalancer/base.py similarity index 100% rename from libcloud/resource/lb/base.py rename to libcloud/loadbalancer/base.py diff --git a/libcloud/resource/lb/drivers/__init__.py b/libcloud/loadbalancer/drivers/__init__.py similarity index 100% rename from libcloud/resource/lb/drivers/__init__.py rename to libcloud/loadbalancer/drivers/__init__.py diff --git a/libcloud/resource/lb/drivers/gogrid.py b/libcloud/loadbalancer/drivers/gogrid.py similarity index 97% rename from libcloud/resource/lb/drivers/gogrid.py rename to libcloud/loadbalancer/drivers/gogrid.py index 1e41b5b388..a22b8ad235 100644 --- a/libcloud/resource/lb/drivers/gogrid.py +++ b/libcloud/loadbalancer/drivers/gogrid.py @@ -23,8 +23,8 @@ from libcloud.common.types import LibcloudError from libcloud.common.gogrid import GoGridConnection, BaseGoGridDriver -from libcloud.resource.lb.base import LB, LBNode, LBDriver -from libcloud.resource.lb.types import Provider, LBState, LibcloudLBImmutableError +from libcloud.loadbalancer.base import LB, LBNode, LBDriver +from libcloud.loadbalancer.types import Provider, LBState, LibcloudLBImmutableError class GoGridLBDriver(BaseGoGridDriver, LBDriver): diff --git a/libcloud/resource/lb/drivers/rackspace.py b/libcloud/loadbalancer/drivers/rackspace.py similarity index 97% rename from libcloud/resource/lb/drivers/rackspace.py rename to libcloud/loadbalancer/drivers/rackspace.py index 6e645357ab..a668f09564 100644 --- a/libcloud/resource/lb/drivers/rackspace.py +++ b/libcloud/loadbalancer/drivers/rackspace.py @@ -21,8 +21,8 @@ import simplejson from libcloud.common.base import Response -from libcloud.resource.lb.base import LB, LBNode, LBDriver -from libcloud.resource.lb.types import Provider, LBState +from libcloud.loadbalancer.base import LB, LBNode, LBDriver +from libcloud.loadbalancer.types import Provider, LBState from libcloud.common.rackspace import (AUTH_HOST_US, RackspaceBaseConnection) diff --git a/libcloud/resource/lb/providers.py b/libcloud/loadbalancer/providers.py similarity index 84% rename from libcloud/resource/lb/providers.py rename to libcloud/loadbalancer/providers.py index 48162c48ce..63a99ef45d 100644 --- a/libcloud/resource/lb/providers.py +++ b/libcloud/loadbalancer/providers.py @@ -14,7 +14,7 @@ # limitations under the License. from libcloud.utils import get_driver as get_provider_driver -from libcloud.resource.lb.types import Provider +from libcloud.loadbalancer.types import Provider __all__ = [ "Provider", @@ -24,9 +24,9 @@ DRIVERS = { Provider.RACKSPACE: - ('libcloud.resource.lb.drivers.rackspace', 'RackspaceLBDriver'), + ('libcloud.loadbalancer.drivers.rackspace', 'RackspaceLBDriver'), Provider.GOGRID: - ('libcloud.resource.lb.drivers.gogrid', 'GoGridLBDriver'), + ('libcloud.loadbalancer.drivers.gogrid', 'GoGridLBDriver'), } def get_driver(provider): diff --git a/libcloud/resource/lb/types.py b/libcloud/loadbalancer/types.py similarity index 100% rename from libcloud/resource/lb/types.py rename to libcloud/loadbalancer/types.py diff --git a/setup.py b/setup.py index 5878a35d67..3a21bb46bb 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ HTML_VIEWSOURCE_BASE = 'https://svn.apache.org/viewvc/incubator/libcloud/trunk' PROJECT_BASE_DIR = 'http://incubator.apache.org/libcloud/' -TEST_PATHS = [ 'test', 'test/compute', 'test/storage' ] +TEST_PATHS = [ 'test', 'test/compute', 'test/storage' , 'test/loadbalancer'] DOC_TEST_MODULES = [ 'libcloud.compute.drivers.dummy', 'libcloud.storage.drivers.dummy' ] @@ -150,9 +150,8 @@ def run(self): 'libcloud.compute', 'libcloud.compute.drivers', 'libcloud.drivers', - 'libcloud.resource', - 'libcloud.resource.lb', - 'libcloud.resource.lb.drivers', + 'libcloud.loadbalancer', + 'libcloud.loadbalancer.drivers', ], package_dir={ 'libcloud': 'libcloud', diff --git a/test/file_fixtures.py b/test/file_fixtures.py index 01ee3c76ab..578b139b03 100644 --- a/test/file_fixtures.py +++ b/test/file_fixtures.py @@ -20,7 +20,7 @@ FIXTURES_ROOT = { 'compute': 'compute/fixtures', 'storage': 'storage/fixtures', - 'resource': 'resource/fixtures', + 'loadbalancer': 'loadbalancer/fixtures', } class FileFixtures(object): @@ -46,7 +46,7 @@ def __init__(self, sub_dir=''): super(StorageFileFixtures, self).__init__(fixtures_type='storage', sub_dir=sub_dir) -class ResourceFileFixtures(FileFixtures): +class LoadBalancerFileFixtures(FileFixtures): def __init__(self, sub_dir=''): - super(ResourceFileFixtures, self).__init__(fixtures_type='resource', + super(LoadBalancerFileFixtures, self).__init__(fixtures_type='loadbalancer', sub_dir=sub_dir) diff --git a/test/loadbalancer/__init__.py b/test/loadbalancer/__init__.py new file mode 100644 index 0000000000..ae1e83eeb3 --- /dev/null +++ b/test/loadbalancer/__init__.py @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/resource/fixtures/lb/gogrid/ip_list.json b/test/loadbalancer/fixtures/gogrid/ip_list.json similarity index 100% rename from test/resource/fixtures/lb/gogrid/ip_list.json rename to test/loadbalancer/fixtures/gogrid/ip_list.json diff --git a/test/resource/fixtures/lb/gogrid/loadbalancer_add.json b/test/loadbalancer/fixtures/gogrid/loadbalancer_add.json similarity index 100% rename from test/resource/fixtures/lb/gogrid/loadbalancer_add.json rename to test/loadbalancer/fixtures/gogrid/loadbalancer_add.json diff --git a/test/resource/fixtures/lb/gogrid/loadbalancer_edit.json b/test/loadbalancer/fixtures/gogrid/loadbalancer_edit.json similarity index 100% rename from test/resource/fixtures/lb/gogrid/loadbalancer_edit.json rename to test/loadbalancer/fixtures/gogrid/loadbalancer_edit.json diff --git a/test/resource/fixtures/lb/gogrid/loadbalancer_get.json b/test/loadbalancer/fixtures/gogrid/loadbalancer_get.json similarity index 100% rename from test/resource/fixtures/lb/gogrid/loadbalancer_get.json rename to test/loadbalancer/fixtures/gogrid/loadbalancer_get.json diff --git a/test/resource/fixtures/lb/gogrid/loadbalancer_list.json b/test/loadbalancer/fixtures/gogrid/loadbalancer_list.json similarity index 100% rename from test/resource/fixtures/lb/gogrid/loadbalancer_list.json rename to test/loadbalancer/fixtures/gogrid/loadbalancer_list.json diff --git a/test/resource/fixtures/lb/rackspace/v1_slug_loadbalancers.json b/test/loadbalancer/fixtures/rackspace/v1_slug_loadbalancers.json similarity index 100% rename from test/resource/fixtures/lb/rackspace/v1_slug_loadbalancers.json rename to test/loadbalancer/fixtures/rackspace/v1_slug_loadbalancers.json diff --git a/test/resource/fixtures/lb/rackspace/v1_slug_loadbalancers_8290.json b/test/loadbalancer/fixtures/rackspace/v1_slug_loadbalancers_8290.json similarity index 100% rename from test/resource/fixtures/lb/rackspace/v1_slug_loadbalancers_8290.json rename to test/loadbalancer/fixtures/rackspace/v1_slug_loadbalancers_8290.json diff --git a/test/resource/fixtures/lb/rackspace/v1_slug_loadbalancers_8290_nodes.json b/test/loadbalancer/fixtures/rackspace/v1_slug_loadbalancers_8290_nodes.json similarity index 100% rename from test/resource/fixtures/lb/rackspace/v1_slug_loadbalancers_8290_nodes.json rename to test/loadbalancer/fixtures/rackspace/v1_slug_loadbalancers_8290_nodes.json diff --git a/test/resource/fixtures/lb/rackspace/v1_slug_loadbalancers_8290_nodes_post.json b/test/loadbalancer/fixtures/rackspace/v1_slug_loadbalancers_8290_nodes_post.json similarity index 100% rename from test/resource/fixtures/lb/rackspace/v1_slug_loadbalancers_8290_nodes_post.json rename to test/loadbalancer/fixtures/rackspace/v1_slug_loadbalancers_8290_nodes_post.json diff --git a/test/resource/fixtures/lb/rackspace/v1_slug_loadbalancers_post.json b/test/loadbalancer/fixtures/rackspace/v1_slug_loadbalancers_post.json similarity index 100% rename from test/resource/fixtures/lb/rackspace/v1_slug_loadbalancers_post.json rename to test/loadbalancer/fixtures/rackspace/v1_slug_loadbalancers_post.json diff --git a/test/resource/lb/test_gogrid.py b/test/loadbalancer/test_gogrid.py similarity index 94% rename from test/resource/lb/test_gogrid.py rename to test/loadbalancer/test_gogrid.py index af94c63bc4..d95ce5f2d6 100644 --- a/test/resource/lb/test_gogrid.py +++ b/test/loadbalancer/test_gogrid.py @@ -3,11 +3,11 @@ import sys import unittest -from libcloud.resource.lb.base import LB, LBNode -from libcloud.resource.lb.drivers.gogrid import GoGridLBDriver +from libcloud.loadbalancer.base import LB, LBNode +from libcloud.loadbalancer.drivers.gogrid import GoGridLBDriver from test import MockHttp, MockRawResponse -from test.file_fixtures import ResourceFileFixtures +from test.file_fixtures import LoadBalancerFileFixtures class GoGridTests(unittest.TestCase): @@ -76,7 +76,7 @@ def test_balancer_detach_node(self): self.assertTrue(ret) class GoGridLBMockHttp(MockHttp): - fixtures = ResourceFileFixtures(os.path.join('lb', 'gogrid')) + fixtures = LoadBalancerFileFixtures('gogrid') def _api_grid_loadbalancer_list(self, method, url, body, headers): body = self.fixtures.load('loadbalancer_list.json') diff --git a/test/resource/lb/test_rackspace.py b/test/loadbalancer/test_rackspace.py similarity index 95% rename from test/resource/lb/test_rackspace.py rename to test/loadbalancer/test_rackspace.py index 91d6cd263a..1121571a6b 100644 --- a/test/resource/lb/test_rackspace.py +++ b/test/loadbalancer/test_rackspace.py @@ -3,11 +3,11 @@ import sys import unittest -from libcloud.resource.lb.base import LB, LBNode -from libcloud.resource.lb.drivers.rackspace import RackspaceLBDriver +from libcloud.loadbalancer.base import LB, LBNode +from libcloud.loadbalancer.drivers.rackspace import RackspaceLBDriver from test import MockHttp, MockRawResponse -from test.file_fixtures import ResourceFileFixtures +from test.file_fixtures import LoadBalancerFileFixtures class RackspaceLBTests(unittest.TestCase): @@ -72,7 +72,7 @@ def test_balancer_detach_node(self): self.assertTrue(ret) class RackspaceLBMockHttp(MockHttp): - fixtures = ResourceFileFixtures(os.path.join('lb', 'rackspace')) + fixtures = LoadBalancerFileFixtures('rackspace') def _v1_0(self, method, url, body, headers): headers = {'x-server-management-url': 'https://servers.api.rackspacecloud.com/v1.0/slug', From df2f02601610a2948a0cc4035decca39be6c349f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 14 May 2011 09:14:55 +0000 Subject: [PATCH 02/90] Simply deploy_node a bit and actually retry the process if it fails. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1102979 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/compute/base.py | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/libcloud/compute/base.py b/libcloud/compute/base.py index 65abc932ba..7d6f502bc5 100644 --- a/libcloud/compute/base.py +++ b/libcloud/compute/base.py @@ -546,29 +546,28 @@ def deploy_node(self, **kwargs): port=ssh_port, username=ssh_username, password=password, timeout=kwargs.get('ssh_timeout', None)) - laste = None + while time.time() < end: - laste = None try: client.connect() - break except (IOError, socket.gaierror, socket.error), e: - laste = e + # Retry if a connection is refused or timeout + # occured + client.close() time.sleep(WAIT_PERIOD) - if laste is not None: - raise e - - tries = 3 - while tries >= 0: - try: - n = kwargs["deploy"].run(node, client) - client.close() - break - except Exception, e: - tries -= 1 - if tries == 0: - raise - client.connect() + continue + + max_tries, tries = 3, 0 + while tries < max_tries: + try: + n = kwargs["deploy"].run(node, client) + client.close() + raise + except Exception, e: + tries += 1 + if tries >= max_tries: + raise DeploymentError(node, + 'Failed after %d tries' % (max_tries)) except DeploymentError: raise From 727dca141755fea521493671c74f5d6f39ef2bb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 14 May 2011 09:16:24 +0000 Subject: [PATCH 03/90] If a ssh timeout is not provided default to 20 seconds. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1102981 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/compute/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libcloud/compute/base.py b/libcloud/compute/base.py index 7d6f502bc5..866feac823 100644 --- a/libcloud/compute/base.py +++ b/libcloud/compute/base.py @@ -541,11 +541,12 @@ def deploy_node(self, **kwargs): ssh_username = kwargs.get('ssh_username', 'root') ssh_port = kwargs.get('ssh_port', 22) + ssh_timeout = kwargs.get('ssh_timeout', 20) client = SSHClient(hostname=node.public_ip[0], port=ssh_port, username=ssh_username, password=password, - timeout=kwargs.get('ssh_timeout', None)) + timeout=ssh_timeout) while time.time() < end: try: From bd39a987d1c68a0c3c50411b0e758e7e60c0e89e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 14 May 2011 09:24:29 +0000 Subject: [PATCH 04/90] Add LBAlgorithm class and set DEFAULT_ALGORITHM variable to ROUND_ROBIN. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1102982 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/base.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/libcloud/loadbalancer/base.py b/libcloud/loadbalancer/base.py index a293cd217d..ed43d8c147 100644 --- a/libcloud/loadbalancer/base.py +++ b/libcloud/loadbalancer/base.py @@ -19,6 +19,7 @@ "LBNode", "LB", "LBDriver", + "LBAlgorithm" ] class LBNode(object): @@ -32,6 +33,12 @@ def __repr__(self): return ('' % (self.id, self.ip, self.port)) +class LBAlgorithm(object): + RANDOM = 0 + ROUND_ROBIN = 1 + LEAST_CONNECTIONS = 2 + +DEFAULT_ALGORITHM = LBAlgorithm.ROUND_ROBIN class LB(object): """ From 478f89107f288089b7385285c99071eb0a5435dd Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Sat, 14 May 2011 09:46:05 +0000 Subject: [PATCH 05/90] Rename 'node{,s}' in the load balancer API to be 'member{,s}', as this is less confusing terminology git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1102988 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/base.py | 44 ++++++++-------- libcloud/loadbalancer/drivers/gogrid.py | 58 +++++++++++----------- libcloud/loadbalancer/drivers/rackspace.py | 36 +++++++------- test/loadbalancer/test_gogrid.py | 32 ++++++------ test/loadbalancer/test_rackspace.py | 28 +++++------ 5 files changed, 99 insertions(+), 99 deletions(-) diff --git a/libcloud/loadbalancer/base.py b/libcloud/loadbalancer/base.py index ed43d8c147..5c5b092fbf 100644 --- a/libcloud/loadbalancer/base.py +++ b/libcloud/loadbalancer/base.py @@ -16,13 +16,13 @@ from libcloud.common.base import ConnectionKey __all__ = [ - "LBNode", + "LBMember", "LB", "LBDriver", "LBAlgorithm" ] -class LBNode(object): +class LBMember(object): def __init__(self, id, ip, port): self.id = str(id) if id else None @@ -30,7 +30,7 @@ def __init__(self, id, ip, port): self.port = port def __repr__(self): - return ('' % (self.id, + return ('' % (self.id, self.ip, self.port)) class LBAlgorithm(object): @@ -53,14 +53,14 @@ def __init__(self, id, name, state, ip, port, driver): self.port = port self.driver = driver - def attach_node(self, **kwargs): - return self.driver.balancer_attach_node(self, **kwargs) + def attach_member(self, **kwargs): + return self.driver.balancer_attach_member(self, **kwargs) - def detach_node(self, node): - return self.driver.balancer_detach_node(self, node) + def detach_member(self, member): + return self.driver.balancer_detach_member(self, member) - def list_nodes(self): - return self.driver.balancer_list_nodes(self) + def list_members(self): + return self.driver.balancer_list_members(self) def __repr__(self): return ('' % (self.id, @@ -110,7 +110,7 @@ def create_balancer(self, **kwargs): @type name: C{str} @keyword port: Port the load balancer should listen on (required) @type port: C{str} - @keyword nodes: C{list} of L{LBNode}s to attach to balancer + @keyword members: C{list} of L{LBNode}s to attach to balancer @type: C{list} of L{LBNode}s """ @@ -145,38 +145,38 @@ def balancer_detail(self, **kwargs): raise NotImplementedError, \ 'balancer_detail not implemented for this driver' - def balancer_attach_node(self, balancer, **kwargs): + def balancer_attach_member(self, balancer, **kwargs): """ - Attach a node to balancer + Attach a member to balancer - @keyword ip: IP address of a node + @keyword ip: IP address of a member @type ip: C{str} - @keyword port: port that services we're balancing listens on on the node + @keyword port: port that services we're balancing listens on on the member @keyword port: C{str} """ raise NotImplementedError, \ - 'balancer_attach_node not implemented for this driver' + 'balancer_attach_member not implemented for this driver' - def balancer_detach_node(self, balancer, node): + def balancer_detach_member(self, balancer, member): """ - Detach node from balancer + Detach member from balancer - @return: C{bool} True if node detach was successful, otherwise False + @return: C{bool} True if member detach was successful, otherwise False """ raise NotImplementedError, \ - 'balancer_detach_node not implemented for this driver' + 'balancer_detach_member not implemented for this driver' - def balancer_list_nodes(self, balancer): + def balancer_list_members(self, balancer): """ - Return list of nodes attached to balancer + Return list of members attached to balancer @return: C{list} of L{LBNode}s """ raise NotImplementedError, \ - 'balancer_list_nodes not implemented for this driver' + 'balancer_list_members not implemented for this driver' diff --git a/libcloud/loadbalancer/drivers/gogrid.py b/libcloud/loadbalancer/drivers/gogrid.py index a22b8ad235..5afdcc089d 100644 --- a/libcloud/loadbalancer/drivers/gogrid.py +++ b/libcloud/loadbalancer/drivers/gogrid.py @@ -23,7 +23,7 @@ from libcloud.common.types import LibcloudError from libcloud.common.gogrid import GoGridConnection, BaseGoGridDriver -from libcloud.loadbalancer.base import LB, LBNode, LBDriver +from libcloud.loadbalancer.base import LB, LBMember, LBDriver from libcloud.loadbalancer.types import Provider, LBState, LibcloudLBImmutableError @@ -43,12 +43,12 @@ def list_balancers(self): def ex_create_balancer_nowait(self, **kwargs): name = kwargs['name'] port = kwargs['port'] - nodes = kwargs['nodes'] + members = kwargs['members'] params = {'name': name, 'virtualip.ip': self._get_first_ip(), 'virtualip.port': port} - params.update(self._nodes_to_params(nodes)) + params.update(self._members_to_params(members)) resp = self.connection.request('/api/grid/loadbalancer/add', method='GET', @@ -108,41 +108,41 @@ def balancer_detail(self, **kwargs): return self._to_balancers(resp.object)[0] - def balancer_attach_node(self, balancer, **kwargs): + def balancer_attach_member(self, balancer, **kwargs): ip = kwargs['ip'] port = kwargs['port'] - nodes = self.balancer_list_nodes(balancer) - nodes.append(LBNode(None, ip, port)) + members = self.balancer_list_members(balancer) + members.append(LBMember(None, ip, port)) params = {"id": balancer.id} - params.update(self._nodes_to_params(nodes)) + params.update(self._members_to_params(members)) - resp = self._update_node(params) + resp = self._update_balancer(params) - return [ node for node in - self._to_nodes(resp.object["list"][0]["realiplist"]) - if node.ip == ip ][0] + return [ member for member in + self._to_members(resp.object["list"][0]["realiplist"]) + if member.ip == ip ][0] - def balancer_detach_node(self, balancer, node): - nodes = self.balancer_list_nodes(balancer) + def balancer_detach_member(self, balancer, member): + members = self.balancer_list_members(balancer) - remaining_nodes = [n for n in nodes if n.id != node.id] + remaining_members = [n for n in members if n.id != member.id] params = {"id": balancer.id} - params.update(self._nodes_to_params(remaining_nodes)) + params.update(self._members_to_params(remaining_members)) - resp = self._update_node(params) + resp = self._update_balancer(params) return resp.status == 200 - def balancer_list_nodes(self, balancer): + def balancer_list_members(self, balancer): resp = self.connection.request('/api/grid/loadbalancer/get', params={'id': balancer.id}) - return self._to_nodes(resp.object["list"][0]["realiplist"]) + return self._to_members(resp.object["list"][0]["realiplist"]) - def _update_node(self, params): + def _update_balancer(self, params): try: return self.connection.request('/api/grid/loadbalancer/edit', method='POST', @@ -153,9 +153,9 @@ def _update_node(self, params): raise LibcloudError(value='Exception: %s' % str(err), driver=self) - def _nodes_to_params(self, nodes): + def _members_to_params(self, members): """ - Helper method to convert list of L{LBNode} objects + Helper method to convert list of L{LBMember} objects to GET params. """ @@ -163,9 +163,9 @@ def _nodes_to_params(self, nodes): params = {} i = 0 - for node in nodes: - params["realiplist.%s.ip" % i] = node.ip - params["realiplist.%s.port" % i] = node.port + for member in members: + params["realiplist.%s.ip" % i] = member.ip + params["realiplist.%s.port" % i] = member.port i += 1 return params @@ -183,11 +183,11 @@ def _to_balancer(self, el): driver=self.connection.driver) return lb - def _to_nodes(self, object): - return [ self._to_node(el) for el in object ] + def _to_members(self, object): + return [ self._to_member(el) for el in object ] - def _to_node(self, el): - lbnode = LBNode(id=el["ip"]["id"], + def _to_member(self, el): + member = LBMember(id=el["ip"]["id"], ip=el["ip"]["ip"], port=el["port"]) - return lbnode + return member diff --git a/libcloud/loadbalancer/drivers/rackspace.py b/libcloud/loadbalancer/drivers/rackspace.py index a668f09564..002fa43230 100644 --- a/libcloud/loadbalancer/drivers/rackspace.py +++ b/libcloud/loadbalancer/drivers/rackspace.py @@ -21,7 +21,7 @@ import simplejson from libcloud.common.base import Response -from libcloud.loadbalancer.base import LB, LBNode, LBDriver +from libcloud.loadbalancer.base import LB, LBMember, LBDriver from libcloud.loadbalancer.types import Provider, LBState from libcloud.common.rackspace import (AUTH_HOST_US, RackspaceBaseConnection) @@ -79,16 +79,16 @@ def list_balancers(self): def create_balancer(self, **kwargs): name = kwargs['name'] port = kwargs['port'] - nodes = kwargs['nodes'] + members = kwargs['members'] balancer_object = {"loadBalancer": {"name": name, "port": port, "protocol": "HTTP", "virtualIps": [{"type": "PUBLIC"}], - "nodes": [{"address": node.ip, - "port": node.port, - "condition": "ENABLED"} for node in nodes], + "nodes": [{"address": member.ip, + "port": member.port, + "condition": "ENABLED"} for member in members], } } @@ -114,11 +114,11 @@ def balancer_detail(self, **kwargs): return self._to_balancer(resp.object["loadBalancer"]) - def balancer_attach_node(self, balancer, **kwargs): + def balancer_attach_member(self, balancer, **kwargs): ip = kwargs['ip'] port = kwargs['port'] - node_object = {"nodes": + member_object = {"nodes": [{"port": port, "address": ip, "condition": "ENABLED"}] @@ -126,18 +126,18 @@ def balancer_attach_node(self, balancer, **kwargs): uri = '/loadbalancers/%s/nodes' % (balancer.id) resp = self.connection.request(uri, method='POST', - data=json.dumps(node_object)) - return self._to_nodes(resp.object)[0] + data=json.dumps(member_object)) + return self._to_members(resp.object)[0] - def balancer_detach_node(self, balancer, node): - uri = '/loadbalancers/%s/nodes/%s' % (balancer.id, node.id) + def balancer_detach_member(self, balancer, member): + uri = '/loadbalancers/%s/nodes/%s' % (balancer.id, member.id) resp = self.connection.request(uri, method='DELETE') return resp.status == 202 - def balancer_list_nodes(self, balancer): + def balancer_list_members(self, balancer): uri = '/loadbalancers/%s/nodes' % (balancer.id) - return self._to_nodes( + return self._to_members( self.connection.request(uri).object) def _to_balancers(self, object): @@ -153,11 +153,11 @@ def _to_balancer(self, el): driver=self.connection.driver) return lb - def _to_nodes(self, object): - return [ self._to_node(el) for el in object["nodes"] ] + def _to_members(self, object): + return [ self._to_member(el) for el in object["nodes"] ] - def _to_node(self, el): - lbnode = LBNode(id=el["id"], + def _to_member(self, el): + lbmember = LBMember(id=el["id"], ip=el["address"], port=el["port"]) - return lbnode + return lbmember diff --git a/test/loadbalancer/test_gogrid.py b/test/loadbalancer/test_gogrid.py index d95ce5f2d6..1b869b54d4 100644 --- a/test/loadbalancer/test_gogrid.py +++ b/test/loadbalancer/test_gogrid.py @@ -3,7 +3,7 @@ import sys import unittest -from libcloud.loadbalancer.base import LB, LBNode +from libcloud.loadbalancer.base import LB, LBMember from libcloud.loadbalancer.drivers.gogrid import GoGridLBDriver from test import MockHttp, MockRawResponse @@ -29,8 +29,8 @@ def test_list_balancers(self): def test_create_balancer(self): balancer = self.driver.create_balancer(name='test2', port=80, - nodes=(LBNode(None, '10.1.0.10', 80), - LBNode(None, '10.1.0.11', 80)) + members=(LBMember(None, '10.1.0.10', 80), + LBMember(None, '10.1.0.11', 80)) ) self.assertEquals(balancer.name, 'test2') @@ -48,30 +48,30 @@ def test_balancer_detail(self): self.assertEquals(balancer.name, 'test2') self.assertEquals(balancer.id, '23530') - def test_balancer_list_nodes(self): + def test_balancer_list_members(self): balancer = self.driver.balancer_detail(balancer_id='23530') - nodes = balancer.list_nodes() + members = balancer.list_members() - expected_nodes = set([u'10.0.0.78:80', u'10.0.0.77:80', + expected_members = set([u'10.0.0.78:80', u'10.0.0.77:80', u'10.0.0.76:80']) - self.assertEquals(len(nodes), 3) - self.assertEquals(expected_nodes, - set(["%s:%s" % (node.ip, node.port) for node in nodes])) + self.assertEquals(len(members), 3) + self.assertEquals(expected_members, + set(["%s:%s" % (member.ip, member.port) for member in members])) - def test_balancer_attach_node(self): + def test_balancer_attach_member(self): balancer = LB(23530, None, None, None, None, None) - node = self.driver.balancer_attach_node(balancer, + member = self.driver.balancer_attach_member(balancer, ip='10.0.0.75', port='80') - self.assertEquals(node.ip, '10.0.0.75') - self.assertEquals(node.port, 80) + self.assertEquals(member.ip, '10.0.0.75') + self.assertEquals(member.port, 80) - def test_balancer_detach_node(self): + def test_balancer_detach_member(self): balancer = LB(23530, None, None, None, None, None) - node = self.driver.balancer_list_nodes(balancer)[0] + member = self.driver.balancer_list_members(balancer)[0] - ret = self.driver.balancer_detach_node(balancer, node) + ret = self.driver.balancer_detach_member(balancer, member) self.assertTrue(ret) diff --git a/test/loadbalancer/test_rackspace.py b/test/loadbalancer/test_rackspace.py index 1121571a6b..ed485c03be 100644 --- a/test/loadbalancer/test_rackspace.py +++ b/test/loadbalancer/test_rackspace.py @@ -3,7 +3,7 @@ import sys import unittest -from libcloud.loadbalancer.base import LB, LBNode +from libcloud.loadbalancer.base import LB, LBMember from libcloud.loadbalancer.drivers.rackspace import RackspaceLBDriver from test import MockHttp, MockRawResponse @@ -29,8 +29,8 @@ def test_list_balancers(self): def test_create_balancer(self): balancer = self.driver.create_balancer(name='test2', port=80, - nodes=(LBNode(None, '10.1.0.10', 80), - LBNode(None, '10.1.0.11', 80)) + members=(LBMember(None, '10.1.0.10', 80), + LBMember(None, '10.1.0.11', 80)) ) self.assertEquals(balancer.name, 'test2') @@ -48,26 +48,26 @@ def test_balancer_detail(self): self.assertEquals(balancer.name, 'test2') self.assertEquals(balancer.id, '8290') - def test_balancer_list_nodes(self): + def test_balancer_list_members(self): balancer = self.driver.balancer_detail(balancer_id='8290') - nodes = balancer.list_nodes() + members = balancer.list_members() - self.assertEquals(len(nodes), 2) + self.assertEquals(len(members), 2) self.assertEquals(set(['10.1.0.10:80', '10.1.0.11:80']), - set(["%s:%s" % (node.ip, node.port) for node in nodes])) + set(["%s:%s" % (member.ip, member.port) for member in members])) - def test_balancer_attach_node(self): + def test_balancer_attach_member(self): balancer = self.driver.balancer_detail(balancer_id='8290') - node = balancer.attach_node(ip='10.1.0.12', port='80') + member = balancer.attach_member(ip='10.1.0.12', port='80') - self.assertEquals(node.ip, '10.1.0.12') - self.assertEquals(node.port, 80) + self.assertEquals(member.ip, '10.1.0.12') + self.assertEquals(member.port, 80) - def test_balancer_detach_node(self): + def test_balancer_detach_member(self): balancer = self.driver.balancer_detail(balancer_id='8290') - node = balancer.list_nodes()[0] + member = balancer.list_members()[0] - ret = balancer.detach_node(node) + ret = balancer.detach_member(member) self.assertTrue(ret) From 6203a80af1f3941af7aa1a4b0b4a9e50bc93597a Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Sat, 14 May 2011 09:47:21 +0000 Subject: [PATCH 06/90] remove unused variables git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1102989 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/drivers/gogrid.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/libcloud/loadbalancer/drivers/gogrid.py b/libcloud/loadbalancer/drivers/gogrid.py index 5afdcc089d..5d42da08a5 100644 --- a/libcloud/loadbalancer/drivers/gogrid.py +++ b/libcloud/loadbalancer/drivers/gogrid.py @@ -13,14 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import time -try: - import json -except ImportError: - import simplejson - from libcloud.common.types import LibcloudError from libcloud.common.gogrid import GoGridConnection, BaseGoGridDriver from libcloud.loadbalancer.base import LB, LBMember, LBDriver From 38ccb6addf7f2ac096cb235e9518f03ed252b3bb Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Sat, 14 May 2011 09:50:16 +0000 Subject: [PATCH 07/90] fix simplejson fallback git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1102990 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/drivers/rackspace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcloud/loadbalancer/drivers/rackspace.py b/libcloud/loadbalancer/drivers/rackspace.py index 002fa43230..3ebecdefcc 100644 --- a/libcloud/loadbalancer/drivers/rackspace.py +++ b/libcloud/loadbalancer/drivers/rackspace.py @@ -18,7 +18,7 @@ try: import json except ImportError: - import simplejson + import simplejson as json from libcloud.common.base import Response from libcloud.loadbalancer.base import LB, LBMember, LBDriver From b95fc39de132655d207164f4e88870b63d4f2985 Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Sat, 14 May 2011 09:50:29 +0000 Subject: [PATCH 08/90] remove unused imports git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1102991 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/storage/drivers/cloudfiles.py | 1 - 1 file changed, 1 deletion(-) diff --git a/libcloud/storage/drivers/cloudfiles.py b/libcloud/storage/drivers/cloudfiles.py index b4c8ff0197..6783ffc6c2 100644 --- a/libcloud/storage/drivers/cloudfiles.py +++ b/libcloud/storage/drivers/cloudfiles.py @@ -21,7 +21,6 @@ except: import simplejson as json -from libcloud.utils import fixxpath, findtext, in_development_warning from libcloud.utils import read_in_chunks from libcloud.common.types import MalformedResponseError, LibcloudError from libcloud.common.base import Response From 64b09eeb92b343000f4d7ec6ba9694d72fba003b Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Sat, 14 May 2011 09:57:45 +0000 Subject: [PATCH 09/90] Have attach_member actually take a LBMember object as its parameter, rather than a raw ip/port git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1102994 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/base.py | 4 ++-- libcloud/loadbalancer/drivers/gogrid.py | 11 ++++------- libcloud/loadbalancer/drivers/rackspace.py | 6 +++--- test/loadbalancer/test_gogrid.py | 2 +- test/loadbalancer/test_rackspace.py | 2 +- 5 files changed, 11 insertions(+), 14 deletions(-) diff --git a/libcloud/loadbalancer/base.py b/libcloud/loadbalancer/base.py index 5c5b092fbf..85de16c12e 100644 --- a/libcloud/loadbalancer/base.py +++ b/libcloud/loadbalancer/base.py @@ -53,8 +53,8 @@ def __init__(self, id, name, state, ip, port, driver): self.port = port self.driver = driver - def attach_member(self, **kwargs): - return self.driver.balancer_attach_member(self, **kwargs) + def attach_member(self, member): + return self.driver.balancer_attach_member(self, member) def detach_member(self, member): return self.driver.balancer_detach_member(self, member) diff --git a/libcloud/loadbalancer/drivers/gogrid.py b/libcloud/loadbalancer/drivers/gogrid.py index 5d42da08a5..b895c2fc88 100644 --- a/libcloud/loadbalancer/drivers/gogrid.py +++ b/libcloud/loadbalancer/drivers/gogrid.py @@ -102,12 +102,9 @@ def balancer_detail(self, **kwargs): return self._to_balancers(resp.object)[0] - def balancer_attach_member(self, balancer, **kwargs): - ip = kwargs['ip'] - port = kwargs['port'] - + def balancer_attach_member(self, balancer, member): members = self.balancer_list_members(balancer) - members.append(LBMember(None, ip, port)) + members.append(member) params = {"id": balancer.id} @@ -115,9 +112,9 @@ def balancer_attach_member(self, balancer, **kwargs): resp = self._update_balancer(params) - return [ member for member in + return [ m for m in self._to_members(resp.object["list"][0]["realiplist"]) - if member.ip == ip ][0] + if m.ip == member.ip ][0] def balancer_detach_member(self, balancer, member): members = self.balancer_list_members(balancer) diff --git a/libcloud/loadbalancer/drivers/rackspace.py b/libcloud/loadbalancer/drivers/rackspace.py index 3ebecdefcc..40fd9e5696 100644 --- a/libcloud/loadbalancer/drivers/rackspace.py +++ b/libcloud/loadbalancer/drivers/rackspace.py @@ -114,9 +114,9 @@ def balancer_detail(self, **kwargs): return self._to_balancer(resp.object["loadBalancer"]) - def balancer_attach_member(self, balancer, **kwargs): - ip = kwargs['ip'] - port = kwargs['port'] + def balancer_attach_member(self, balancer, member): + ip = member.ip + port = member.port member_object = {"nodes": [{"port": port, diff --git a/test/loadbalancer/test_gogrid.py b/test/loadbalancer/test_gogrid.py index 1b869b54d4..89ed903b5a 100644 --- a/test/loadbalancer/test_gogrid.py +++ b/test/loadbalancer/test_gogrid.py @@ -62,7 +62,7 @@ def test_balancer_list_members(self): def test_balancer_attach_member(self): balancer = LB(23530, None, None, None, None, None) member = self.driver.balancer_attach_member(balancer, - ip='10.0.0.75', port='80') + LBMember(None, ip='10.0.0.75', port='80')) self.assertEquals(member.ip, '10.0.0.75') self.assertEquals(member.port, 80) diff --git a/test/loadbalancer/test_rackspace.py b/test/loadbalancer/test_rackspace.py index ed485c03be..a07e52dcb2 100644 --- a/test/loadbalancer/test_rackspace.py +++ b/test/loadbalancer/test_rackspace.py @@ -58,7 +58,7 @@ def test_balancer_list_members(self): def test_balancer_attach_member(self): balancer = self.driver.balancer_detail(balancer_id='8290') - member = balancer.attach_member(ip='10.1.0.12', port='80') + member = balancer.attach_member(LBMember(None, ip='10.1.0.12', port='80')) self.assertEquals(member.ip, '10.1.0.12') self.assertEquals(member.port, 80) From 85ac5dfe383b993cdb4a9c057b825e1782eaac73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 14 May 2011 10:09:16 +0000 Subject: [PATCH 10/90] Add two new instance variables and utility methods to the base LBDriver class. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103000 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/base.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/libcloud/loadbalancer/base.py b/libcloud/loadbalancer/base.py index 85de16c12e..1912da9091 100644 --- a/libcloud/loadbalancer/base.py +++ b/libcloud/loadbalancer/base.py @@ -14,6 +14,7 @@ # limitations under the License. from libcloud.common.base import ConnectionKey +from libcloud.common.types import LibcloudError __all__ = [ "LBMember", @@ -76,6 +77,8 @@ class LBDriver(object): """ connectionCls = ConnectionKey + _algorithm_to_value_map = {} + _value_to_algorithm_map = {} def __init__(self, key, secret=None, secure=True): self.key = key @@ -110,6 +113,8 @@ def create_balancer(self, **kwargs): @type name: C{str} @keyword port: Port the load balancer should listen on (required) @type port: C{str} + @keyword algorithm: Load balancing algorithm (defaults to round robin) + @type algorithm: C{LBAlgorithm} @keyword members: C{list} of L{LBNode}s to attach to balancer @type: C{list} of L{LBNode}s @@ -180,3 +185,23 @@ def balancer_list_members(self, balancer): raise NotImplementedError, \ 'balancer_list_members not implemented for this driver' + + def _value_to_algorithm(self, value): + """ + Return C{LBAlgorithm} based on the value. + """ + try: + return self._value_to_algorithm_map[value] + except KeyError: + raise LibcloudError(value='Invalid value: %s' % (value), + driver=self) + + def _algorithm_to_value(self, algorithm): + """ + Return value based in the algorithm (C{LBAlgorithm}). + """ + try: + return self._algorithm_to_value_map[algorithm] + except KeyError: + raise LibcloudError(value='Invalid algorithm: %s' % (algorithm), + driver=self) From f76a5da5d59ed630afd1ee307f34b60c99b18dda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 14 May 2011 10:13:03 +0000 Subject: [PATCH 11/90] Fix formatting. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103002 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/drivers/gogrid.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libcloud/loadbalancer/drivers/gogrid.py b/libcloud/loadbalancer/drivers/gogrid.py index b895c2fc88..de03aad2f4 100644 --- a/libcloud/loadbalancer/drivers/gogrid.py +++ b/libcloud/loadbalancer/drivers/gogrid.py @@ -40,8 +40,8 @@ def ex_create_balancer_nowait(self, **kwargs): members = kwargs['members'] params = {'name': name, - 'virtualip.ip': self._get_first_ip(), - 'virtualip.port': port} + 'virtualip.ip': self._get_first_ip(), + 'virtualip.port': port} params.update(self._members_to_params(members)) resp = self.connection.request('/api/grid/loadbalancer/add', From 4960b18e1a8340f76092f156d29eff8c525f6ece Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Sat, 14 May 2011 10:13:27 +0000 Subject: [PATCH 12/90] Add balancer_attach_compute_node, which takes a normal libcloud compute node object, and attempts to add it to the load balancer as a member git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103003 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/base.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/libcloud/loadbalancer/base.py b/libcloud/loadbalancer/base.py index 1912da9091..6e35c8cb3b 100644 --- a/libcloud/loadbalancer/base.py +++ b/libcloud/loadbalancer/base.py @@ -54,8 +54,8 @@ def __init__(self, id, name, state, ip, port, driver): self.port = port self.driver = driver - def attach_member(self, member): - return self.driver.balancer_attach_member(self, member) + def attach_compute_node(self, node): + return self.driver.balacner_attach_compute_node(node) def detach_member(self, member): return self.driver.balancer_detach_member(self, member) @@ -150,15 +150,24 @@ def balancer_detail(self, **kwargs): raise NotImplementedError, \ 'balancer_detail not implemented for this driver' - def balancer_attach_member(self, balancer, **kwargs): + def balancer_attach_node(self, balacner, node): + """ + Attach a compute node as a member to the load balancer. + + @keyword node: Member to join to the balancer + @type member: C{libcloud.compute.base.Node} + @return {LBMember} Member after joining the balancer. + """ + + return self.attach_member(LBMember(None, node.public_ip[0], balacner.port)) + + def balancer_attach_member(self, balancer, member): """ Attach a member to balancer - @keyword ip: IP address of a member - @type ip: C{str} - @keyword port: port that services we're balancing listens on on the member - @keyword port: C{str} - + @keyword member: Member to join to the balancer + @type member: C{LBMember} + @return {LBMember} Member after joining the balancer. """ raise NotImplementedError, \ From 6837080735fc94e7e4e042a18a92647a4d06603f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 14 May 2011 10:22:20 +0000 Subject: [PATCH 13/90] LBNode -> LBMember in example_lb.py git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103005 13f79535-47bb-0310-9956-ffa450edef68 --- example_lb.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/example_lb.py b/example_lb.py index 0fbd186328..10dc6e72fc 100644 --- a/example_lb.py +++ b/example_lb.py @@ -3,7 +3,7 @@ import os import time -from libcloud.loadbalancer.base import LB, LBNode +from libcloud.loadbalancer.base import LB, LBMember from libcloud.loadbalancer.types import Provider, LBState from libcloud.loadbalancer.providers import get_driver @@ -20,8 +20,8 @@ def main(): new_balancer_name = 'testlb' + os.urandom(4).encode('hex') new_balancer = driver.create_balancer(name=new_balancer_name, port=80, - nodes=(LBNode(None, '192.168.86.1', 80), - LBNode(None, '192.168.86.2', 8080)) + nodes=(LBMember(None, '192.168.86.1', 80), + LBMember(None, '192.168.86.2', 8080)) ) print new_balancer From c1d263f3d7e8a389ca31fabfba0a9803c500442a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 14 May 2011 10:24:06 +0000 Subject: [PATCH 14/90] Rename example_lb.py -> example_loadbalancer.py. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103006 13f79535-47bb-0310-9956-ffa450edef68 --- example_lb.py => example_loadbalancer.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename example_lb.py => example_loadbalancer.py (100%) diff --git a/example_lb.py b/example_loadbalancer.py similarity index 100% rename from example_lb.py rename to example_loadbalancer.py From 9c569a2fe3156dffbe37bc1fefe665b8d70738a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 14 May 2011 10:28:32 +0000 Subject: [PATCH 15/90] Fix a typo and make instance variables uppercase. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103009 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/base.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/libcloud/loadbalancer/base.py b/libcloud/loadbalancer/base.py index 6e35c8cb3b..260b745fa4 100644 --- a/libcloud/loadbalancer/base.py +++ b/libcloud/loadbalancer/base.py @@ -55,7 +55,7 @@ def __init__(self, id, name, state, ip, port, driver): self.driver = driver def attach_compute_node(self, node): - return self.driver.balacner_attach_compute_node(node) + return self.driver.balancer_attach_compute_node(node) def detach_member(self, member): return self.driver.balancer_detach_member(self, member) @@ -77,8 +77,8 @@ class LBDriver(object): """ connectionCls = ConnectionKey - _algorithm_to_value_map = {} - _value_to_algorithm_map = {} + _ALGORITHM_TO_VALUE_MAP = {} + _VALUE_TO_ALGORITHM_MAP = {} def __init__(self, key, secret=None, secure=True): self.key = key @@ -150,7 +150,7 @@ def balancer_detail(self, **kwargs): raise NotImplementedError, \ 'balancer_detail not implemented for this driver' - def balancer_attach_node(self, balacner, node): + def balancer_attach_compute_node(self, balancer, node): """ Attach a compute node as a member to the load balancer. @@ -159,8 +159,8 @@ def balancer_attach_node(self, balacner, node): @return {LBMember} Member after joining the balancer. """ - return self.attach_member(LBMember(None, node.public_ip[0], balacner.port)) - + return self.attach_member(LBMember(None, node.public_ip[0], balancer.port)) + def balancer_attach_member(self, balancer, member): """ Attach a member to balancer @@ -200,7 +200,7 @@ def _value_to_algorithm(self, value): Return C{LBAlgorithm} based on the value. """ try: - return self._value_to_algorithm_map[value] + return self._VALUE_TO_ALGORITHM_MAP[value] except KeyError: raise LibcloudError(value='Invalid value: %s' % (value), driver=self) @@ -210,7 +210,7 @@ def _algorithm_to_value(self, algorithm): Return value based in the algorithm (C{LBAlgorithm}). """ try: - return self._algorithm_to_value_map[algorithm] + return self._ALGORITHM_TO_VALUE_MAP[algorithm] except KeyError: raise LibcloudError(value='Invalid algorithm: %s' % (algorithm), driver=self) From e7c5b1f86e3b2f2105d6c704e29e535065fd9a77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 14 May 2011 10:30:55 +0000 Subject: [PATCH 16/90] Add missing attach_member method. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103010 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libcloud/loadbalancer/base.py b/libcloud/loadbalancer/base.py index 260b745fa4..dd52484114 100644 --- a/libcloud/loadbalancer/base.py +++ b/libcloud/loadbalancer/base.py @@ -57,6 +57,9 @@ def __init__(self, id, name, state, ip, port, driver): def attach_compute_node(self, node): return self.driver.balancer_attach_compute_node(node) + def attach_member(self, member): + return self.driver.balancer_attach_member(self, member) + def detach_member(self, member): return self.driver.balancer_detach_member(self, member) From ee529204e5e956adb067279975e99b6fa231d7fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 14 May 2011 11:13:48 +0000 Subject: [PATCH 17/90] 1. Don't use kwargs 2. Use the algorithm algorithm and pass it to the api git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103039 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/drivers/gogrid.py | 27 ++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/libcloud/loadbalancer/drivers/gogrid.py b/libcloud/loadbalancer/drivers/gogrid.py index de03aad2f4..570d9714ae 100644 --- a/libcloud/loadbalancer/drivers/gogrid.py +++ b/libcloud/loadbalancer/drivers/gogrid.py @@ -16,8 +16,10 @@ import time from libcloud.common.types import LibcloudError +from libcloud.utils import reverse_dict from libcloud.common.gogrid import GoGridConnection, BaseGoGridDriver -from libcloud.loadbalancer.base import LB, LBMember, LBDriver +from libcloud.loadbalancer.base import LB, LBMember, LBDriver, LBAlgorithm +from libcloud.loadbalancer.base import DEFAULT_ALGORITHM from libcloud.loadbalancer.types import Provider, LBState, LibcloudLBImmutableError @@ -29,19 +31,26 @@ class GoGridLBDriver(BaseGoGridDriver, LBDriver): LB_STATE_MAP = { 'On': LBState.RUNNING, 'Unknown': LBState.UNKNOWN } + _VALUE_TO_ALGORITHM_MAP = { + 'round balancer': LBAlgorithm.ROUND_ROBIN, + 'least connection': LBAlgorithm.LEAST_CONNECTIONS + } + _ALGORITHM_TO_VALUE_MAP = reverse_dict(_VALUE_TO_ALGORITHM_MAP) def list_balancers(self): return self._to_balancers( self.connection.request('/api/grid/loadbalancer/list').object) - def ex_create_balancer_nowait(self, **kwargs): - name = kwargs['name'] - port = kwargs['port'] - members = kwargs['members'] + def ex_create_balancer_nowait(self, name, port, algorithm, members): + if not algorithm: + algorithm = DEFAULT_ALGORITHM + else: + algorithm = self._algorithm_to_value(algorithm) params = {'name': name, - 'virtualip.ip': self._get_first_ip(), - 'virtualip.port': port} + 'loadbalancer.type': algorithm, + 'virtualip.ip': self._get_first_ip(), + 'virtualip.port': port} params.update(self._members_to_params(members)) resp = self.connection.request('/api/grid/loadbalancer/add', @@ -49,8 +58,8 @@ def ex_create_balancer_nowait(self, **kwargs): params=params) return self._to_balancers(resp.object)[0] - def create_balancer(self, **kwargs): - balancer = self.ex_create_balancer_nowait(**kwargs) + def create_balancer(self, name, port, algorithm, members): + balancer = self.ex_create_balancer_nowait(name, port, algorithm, members) timeout = 60 * 20 waittime = 0 From e5fb8435b11f12387ecd9c5d362bac6b1e7bd52b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 14 May 2011 11:14:11 +0000 Subject: [PATCH 18/90] Add reverse_dict utility function. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103040 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libcloud/utils.py b/libcloud/utils.py index df2412ffdd..a9ab20beff 100644 --- a/libcloud/utils.py +++ b/libcloud/utils.py @@ -176,6 +176,9 @@ def findattr(element, xpath, namespace): def findall(element, xpath, namespace): return element.findall(fixxpath(xpath=xpath, namespace=namespace)) +def reverse_dict(dictionary): + return dict([ (value, key) for key, value in dictionary.iteritems() ]) + def get_driver(drivers, provider): """ Get a driver. From 5d899faefa876117261ee57d4df55942cbf61537 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 14 May 2011 11:25:02 +0000 Subject: [PATCH 19/90] Don't use kwargs here. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103044 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcloud/loadbalancer/base.py b/libcloud/loadbalancer/base.py index dd52484114..9b7f02c1d1 100644 --- a/libcloud/loadbalancer/base.py +++ b/libcloud/loadbalancer/base.py @@ -108,7 +108,7 @@ def list_balancers(self): raise NotImplementedError, \ 'list_balancers not implemented for this driver' - def create_balancer(self, **kwargs): + def create_balancer(self, name, port, algorithm, members): """ Create a new load balancer instance From d742118a0b20a531585994562f18171f604448c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 14 May 2011 11:25:42 +0000 Subject: [PATCH 20/90] Pass in the algorithm to the create_balancer method. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103045 13f79535-47bb-0310-9956-ffa450edef68 --- test/loadbalancer/test_gogrid.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/loadbalancer/test_gogrid.py b/test/loadbalancer/test_gogrid.py index 89ed903b5a..80cdd6dc4d 100644 --- a/test/loadbalancer/test_gogrid.py +++ b/test/loadbalancer/test_gogrid.py @@ -3,7 +3,7 @@ import sys import unittest -from libcloud.loadbalancer.base import LB, LBMember +from libcloud.loadbalancer.base import LB, LBMember, LBAlgorithm from libcloud.loadbalancer.drivers.gogrid import GoGridLBDriver from test import MockHttp, MockRawResponse @@ -29,6 +29,7 @@ def test_list_balancers(self): def test_create_balancer(self): balancer = self.driver.create_balancer(name='test2', port=80, + algorithm=LBAlgorithm.ROUND_ROBIN, members=(LBMember(None, '10.1.0.10', 80), LBMember(None, '10.1.0.11', 80)) ) From 6e02b62c5a5bfbf9e8f2957d341a0bfc6330efc0 Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Sat, 14 May 2011 11:25:54 +0000 Subject: [PATCH 21/90] Centralize the new compute drivers list in the CHANGES file git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103047 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/CHANGES b/CHANGES index 1e867d01de..1acb1037bf 100644 --- a/CHANGES +++ b/CHANGES @@ -1,16 +1,18 @@ -*- coding: utf-8 -*- Changes with Apache Libcloud 0.4.3 + + *) Add new libcloud.compute drivers for: + - OpenStack [Roman Bogorodskiy] + - Gandi.net [Aymeric Barantal] + - Bluebox [Christian Paredes] + - Nimbus [David LaBissoniere] + *) Add implementation for the following methods to the CloudFiles storage driver: enable_container_cdn(), get_container_cdn_url(), get_object_cdn_url() [Tomaz Muraus] - *) Add OpenStack driver, which is actually an extension for - Rackspace driver that allows to specify custom port and - host of user's OpenStack installation - [Roman Bogorodskiy] - *) Add support for Load Balancing services, available through libcloud.resource.lb. Drivers for Rackspace and GoGrid are included. @@ -24,19 +26,10 @@ Changes with Apache Libcloud 0.4.3 and create a "Name" tag with the value of this argument. [Tomaz Muraus] - *) Add Gandi.net driver. - [Aymeric Barantal] - *) Add extension method for modifying node attributes and changing the node size (EC2 driver). [Tomaz Muraus] - *) Add Bluebox driver. - [Christian Paredes] - - *) Add Nimbus driver. - [David LaBissoniere] - *) Minor fixes to get the library and tests working on Python 2.7 and PyPy. [Tomaz Muraus] From ef2466e25e308cd0abf2ab31073faa506ef53479 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 14 May 2011 11:31:04 +0000 Subject: [PATCH 22/90] Rename the balancer_detail to get_balancer and change the method signature and return value. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103053 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/base.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/libcloud/loadbalancer/base.py b/libcloud/loadbalancer/base.py index 9b7f02c1d1..ad95600154 100644 --- a/libcloud/loadbalancer/base.py +++ b/libcloud/loadbalancer/base.py @@ -136,22 +136,18 @@ def destroy_balancer(self, balancer): raise NotImplementedError, \ 'destroy_balancer not implemented for this driver' - def balancer_detail(self, **kwargs): + def get_balancer(self, balancer_id): """ - Returns a detailed info about load balancer given by - existing L{LB} object or its id + Return a C{LB} object. - @keyword balancer: L{LB} object you already fetched using list method for example - @type balancer: L{LB} @keyword balancer_id: id of a load balancer you want to fetch @type balancer_id: C{str} - @return: L{LB} - + @return: C{LB} """ raise NotImplementedError, \ - 'balancer_detail not implemented for this driver' + 'get_balancer not implemented for this driver' def balancer_attach_compute_node(self, balancer, node): """ From 585219dd21db94e910eabc7236346ff332e23d8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 14 May 2011 11:35:39 +0000 Subject: [PATCH 23/90] Rename the balancer_details method to get_balancer and refactor id. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103056 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/drivers/gogrid.py | 10 +++------- test/loadbalancer/test_gogrid.py | 6 +++--- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/libcloud/loadbalancer/drivers/gogrid.py b/libcloud/loadbalancer/drivers/gogrid.py index 570d9714ae..224165d34a 100644 --- a/libcloud/loadbalancer/drivers/gogrid.py +++ b/libcloud/loadbalancer/drivers/gogrid.py @@ -93,17 +93,13 @@ def destroy_balancer(self, balancer): return resp.status == 200 - def balancer_detail(self, **kwargs): + def get_balancer(self, **kwargs): params = {} try: - params['name'] = kwargs['balancer_name'] + params['name'] = kwargs['ex_balancer_name'] except KeyError: - try: - balancer_id = kwargs['balancer_id'] - except KeyError: - balancer_id = kwargs['balancer'].id - + balancer_id = kwargs['balancer_id'] params['id'] = balancer_id resp = self.connection.request('/api/grid/loadbalancer/get', diff --git a/test/loadbalancer/test_gogrid.py b/test/loadbalancer/test_gogrid.py index 80cdd6dc4d..f7cc557e82 100644 --- a/test/loadbalancer/test_gogrid.py +++ b/test/loadbalancer/test_gogrid.py @@ -43,14 +43,14 @@ def test_destroy_balancer(self): ret = self.driver.destroy_balancer(balancer) self.assertTrue(ret) - def test_balancer_detail(self): - balancer = self.driver.balancer_detail(balancer_id='23530') + def test_get_balancer(self): + balancer = self.driver.get_balancer(balancer_id='23530') self.assertEquals(balancer.name, 'test2') self.assertEquals(balancer.id, '23530') def test_balancer_list_members(self): - balancer = self.driver.balancer_detail(balancer_id='23530') + balancer = self.driver.get_balancer(balancer_id='23530') members = balancer.list_members() expected_members = set([u'10.0.0.78:80', u'10.0.0.77:80', From afc1f46e368c8e997b5964b643e89d79737608d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 14 May 2011 11:39:09 +0000 Subject: [PATCH 24/90] Do the same with the balancer_detail method in the Rackspace LB driver. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103058 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/drivers/rackspace.py | 7 +------ test/loadbalancer/test_rackspace.py | 10 +++++----- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/libcloud/loadbalancer/drivers/rackspace.py b/libcloud/loadbalancer/drivers/rackspace.py index 40fd9e5696..610d4203ed 100644 --- a/libcloud/loadbalancer/drivers/rackspace.py +++ b/libcloud/loadbalancer/drivers/rackspace.py @@ -103,12 +103,7 @@ def destroy_balancer(self, balancer): return resp.status == 202 - def balancer_detail(self, **kwargs): - try: - balancer_id = kwargs['balancer_id'] - except KeyError: - balancer_id = kwargs['balancer'].id - + def get_balancer(self, balancer_id): uri = '/loadbalancers/%s' % (balancer_id) resp = self.connection.request(uri) diff --git a/test/loadbalancer/test_rackspace.py b/test/loadbalancer/test_rackspace.py index a07e52dcb2..cf3d65d93e 100644 --- a/test/loadbalancer/test_rackspace.py +++ b/test/loadbalancer/test_rackspace.py @@ -42,14 +42,14 @@ def test_destroy_balancer(self): ret = self.driver.destroy_balancer(balancer) self.assertTrue(ret) - def test_balancer_detail(self): - balancer = self.driver.balancer_detail(balancer_id='8290') + def test_get_balancer(self): + balancer = self.driver.get_balancer(balancer_id='8290') self.assertEquals(balancer.name, 'test2') self.assertEquals(balancer.id, '8290') def test_balancer_list_members(self): - balancer = self.driver.balancer_detail(balancer_id='8290') + balancer = self.driver.get_balancer(balancer_id='8290') members = balancer.list_members() self.assertEquals(len(members), 2) @@ -57,14 +57,14 @@ def test_balancer_list_members(self): set(["%s:%s" % (member.ip, member.port) for member in members])) def test_balancer_attach_member(self): - balancer = self.driver.balancer_detail(balancer_id='8290') + balancer = self.driver.get_balancer(balancer_id='8290') member = balancer.attach_member(LBMember(None, ip='10.1.0.12', port='80')) self.assertEquals(member.ip, '10.1.0.12') self.assertEquals(member.port, 80) def test_balancer_detach_member(self): - balancer = self.driver.balancer_detail(balancer_id='8290') + balancer = self.driver.get_balancer(balancer_id='8290') member = balancer.list_members()[0] ret = balancer.detach_member(member) From 751dc00080597834a98da8388eeae20dfd7ace02 Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Sat, 14 May 2011 11:39:23 +0000 Subject: [PATCH 25/90] Rewrite changes for 0.5, grouping things by relevant driver git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103059 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES | 90 ++++++++++++++++++++++----------------------------------- 1 file changed, 34 insertions(+), 56 deletions(-) diff --git a/CHANGES b/CHANGES index 1acb1037bf..c040ad6054 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,16 @@ -*- coding: utf-8 -*- -Changes with Apache Libcloud 0.4.3 +Changes with Apache Libcloud 0.5.0 + + *) Add new libcloud.loadbalancers API, with initial support for: + - Rackspace + - GoGrid + [Roman Bogorodskiy] + + *) Add new libcloud.storage API, with initial support for: + - CloudFiles + - Amazon S3 + [Tomaz Muraus] *) Add new libcloud.compute drivers for: - OpenStack [Roman Bogorodskiy] @@ -8,70 +18,38 @@ Changes with Apache Libcloud 0.4.3 - Bluebox [Christian Paredes] - Nimbus [David LaBissoniere] - *) Add implementation for the following methods to the CloudFiles - storage driver: enable_container_cdn(), get_container_cdn_url(), - get_object_cdn_url() + *) Added "pricing" module and improved pricing handling. [Tomaz Muraus] - *) Add support for Load Balancing services, available through - libcloud.resource.lb. Drivers for Rackspace and GoGrid are - included. + *) Updates to the GoGrid compute driver: + - Use API version 1.0. + - Remove sandbox flag. + - Add ex_list_ips() to list IP addresses assigned to the account. + - Implement ex_edit_image method which allows changing image attributes + like name, description and make image public or private. [Roman Bogorodskiy] - *) Update GoGrid driver to use API Version 1.8. Sandbox flag - for servers is no longer used. - [Roman Bogorodskiy] - - *) When creating an EC2 node, don't ignore the name argument - and create a "Name" tag with the value of this argument. - [Tomaz Muraus] - - *) Add extension method for modifying node attributes and - changing the node size (EC2 driver). - [Tomaz Muraus] + *) Updates to the Amazon EC2 compute driver: + - When creating a Node, use the name argument to set a Tag with the + value. [Tomaz Muraus] + - Add extension method for modifying node attributes and changing the + node size. [Tomaz Muraus] + - Add support for the new Amazon Region (Tokyo). [Tomaz Muraus] + - Added ex_create_tags and ex_delete_tags. [Brandon Rhodes] + - Include node Elastic IP addresses in the node public_ip attribute + for the EC2 nodes. [Tomaz Muraus] + - Use ipAddress and privateIpAddress attribute for the EC 2node public + and private ip. [Tomaz Muraus] + - Add ex_describe_addresses method to the EC2 driver. [Tomaz Muraus] + + *) Updates to the Rackspace CloudServers compute driver: + - Add ex_rebuild() and ex_get_node_details() [Andrew Klochkov] + - Expose URI of a Rackspace node to the node meta data. [Paul Querna] *) Minor fixes to get the library and tests working on Python 2.7 and PyPy. [Tomaz Muraus] - *) Added "pricing" module and improved pricing handling. - [Tomaz Muraus] - - *) Add support for the new Amazon Region (Tokyo) - [Tomaz Muraus] - - *) Implement ex_rebuild() and ex_get_node_details() - routines for Rackspace driver. - [Andrew Klochkov] - - *) Implement ex_list_ips() for GoGrid driver to list - IP addresses assigned to the account. Make use of - it in _get_first_ip(). - [Roman Bogorodskiy] - - *) Added ex_create_tags and ex_delete_tags methods to the - EC2 driver. - [Brandon Rhodes] - - *) Include node Elastic IP addresses in the node public_ip - attribute for the EC2 nodes. - [Tomaz Muraus] - - *) Use ipAddress and privateIpAddress attribute for - the EC 2node public and private ip - [Tomaz Muraus] - - *) Add ex_describe_addresses method to the EC2 driver. - [Tomaz Muraus] - - *) Expose URI of a Rackspace node to the node meta data. - [Paul Querna] - - *) Implement ex_edit_image method for GoGrid driver - which allows changing image attributes like name, - description and make image public or private. - [Roman Bogorodskiy] - Changes with Apache Libcloud 0.4.2 *) Fix EC2 create_node to become backward compatible for From fe5dedfc2005b90d2c5d4432ae2637befc89541b Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Sat, 14 May 2011 11:42:19 +0000 Subject: [PATCH 26/90] Sort drivers alphabetically, fixup some formatting git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103061 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/CHANGES b/CHANGES index c040ad6054..69fcffda0e 100644 --- a/CHANGES +++ b/CHANGES @@ -3,20 +3,20 @@ Changes with Apache Libcloud 0.5.0 *) Add new libcloud.loadbalancers API, with initial support for: - - Rackspace - - GoGrid + - GoGrid Load Balancers + - Rackspace Load Balancers [Roman Bogorodskiy] *) Add new libcloud.storage API, with initial support for: - - CloudFiles - Amazon S3 + - Rackspace CloudFiles [Tomaz Muraus] *) Add new libcloud.compute drivers for: - - OpenStack [Roman Bogorodskiy] - - Gandi.net [Aymeric Barantal] - Bluebox [Christian Paredes] + - Gandi.net [Aymeric Barantal] - Nimbus [David LaBissoniere] + - OpenStack [Roman Bogorodskiy] *) Added "pricing" module and improved pricing handling. [Tomaz Muraus] @@ -46,8 +46,7 @@ Changes with Apache Libcloud 0.5.0 - Add ex_rebuild() and ex_get_node_details() [Andrew Klochkov] - Expose URI of a Rackspace node to the node meta data. [Paul Querna] - *) Minor fixes to get the library and tests working on - Python 2.7 and PyPy. + *) Minor fixes to get the library and tests working on Python 2.7 and PyPy. [Tomaz Muraus] Changes with Apache Libcloud 0.4.2 From 632c7bf929d78d62ecb51eecaec6a08751691145 Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Sat, 14 May 2011 11:43:27 +0000 Subject: [PATCH 27/90] Add release date for 0.4.2 git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103062 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 69fcffda0e..e3d0f82058 100644 --- a/CHANGES +++ b/CHANGES @@ -49,7 +49,7 @@ Changes with Apache Libcloud 0.5.0 *) Minor fixes to get the library and tests working on Python 2.7 and PyPy. [Tomaz Muraus] -Changes with Apache Libcloud 0.4.2 +Changes with Apache Libcloud 0.4.2 (Released January 18, 2011) *) Fix EC2 create_node to become backward compatible for NodeLocation. From 8755dc3832a05558928c88722b846cf617691db5 Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Sat, 14 May 2011 12:00:18 +0000 Subject: [PATCH 28/90] Update / sync docs with actual args git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103071 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/storage/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libcloud/storage/base.py b/libcloud/storage/base.py index ebde0e00ee..666f6a1e2a 100644 --- a/libcloud/storage/base.py +++ b/libcloud/storage/base.py @@ -263,7 +263,7 @@ def enable_object_cdn(self, obj): raise NotImplementedError( 'enable_object_cdn not implemented for this driver') - def download_object(self, obj, destination_path, delete_on_failure=True): + def download_object(self, obj, destination_path, overwrite_existing=False, delete_on_failure=True): """ Download an object to the specified destination path. @@ -275,7 +275,7 @@ def download_object(self, obj, destination_path, delete_on_failure=True): incoming file will be saved. @type overwrite_existing: C{bool} - @type overwrite_existing: True to overwrite an existing file. + @type overwrite_existing: True to overwrite an existing file, defaults to False. @type delete_on_failure: C{bool} @param delete_on_failure: True to delete a partially downloaded file if @@ -438,7 +438,7 @@ def _save_object(self, response, obj, destination_path, exists. @type chunk_size: C{int} - @param chunk_size: Optional chunk size (defaults to CHUNK_SIZE) + @param chunk_size: Optional chunk size (defaults to L{libcloud.storage.base.CHUNK_SIZE}, 8kb) @return C{bool} True on success, False otherwise. """ From 940fc1c225abd7f58cee3d1c6ba167dff2560be1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 14 May 2011 13:21:21 +0000 Subject: [PATCH 29/90] Change the create_balancer method signature and add list_protocols method. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103096 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/base.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/libcloud/loadbalancer/base.py b/libcloud/loadbalancer/base.py index ad95600154..7c60692f97 100644 --- a/libcloud/loadbalancer/base.py +++ b/libcloud/loadbalancer/base.py @@ -20,7 +20,8 @@ "LBMember", "LB", "LBDriver", - "LBAlgorithm" + "LBAlgorithm", + "LBProtocol" ] class LBMember(object): @@ -97,6 +98,14 @@ def __init__(self, key, secret=None, secure=True): self.connection.driver = self self.connection.connect() + def list_protocols(self): + """ + Return a list of supported protocols. + """ + + raise NotImplementedError, \ + 'list_protocols not implemented for this driver' + def list_balancers(self): """ List all loadbalancers @@ -108,18 +117,21 @@ def list_balancers(self): raise NotImplementedError, \ 'list_balancers not implemented for this driver' - def create_balancer(self, name, port, algorithm, members): + def create_balancer(self, name, port, protocol, algorithm, members): """ Create a new load balancer instance @keyword name: Name of the new load balancer (required) @type name: C{str} - @keyword port: Port the load balancer should listen on (required) + @keyword members: C{list} ofL{LBNode}s to attach to balancer + @type: C{list} of L{LBNode}s + @keyword protocol: Loadbalancer protocol, defaults to http. + @type: C{str} + @keyword port: Port the load balancer should listen on, defaults to 80 @type port: C{str} - @keyword algorithm: Load balancing algorithm (defaults to round robin) + @keyword algorithm: Load balancing algorithm, defaults to + LBAlgorithm.ROUND_ROBIN @type algorithm: C{LBAlgorithm} - @keyword members: C{list} of L{LBNode}s to attach to balancer - @type: C{list} of L{LBNode}s """ From 6198094b3dcb7ac091e427bef0999354421d50da Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Sat, 14 May 2011 13:22:24 +0000 Subject: [PATCH 30/90] Remove the LB prefix to most libcloud.loadbalancer.* types git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103098 13f79535-47bb-0310-9956-ffa450edef68 --- example_loadbalancer.py | 12 ++++++------ libcloud/loadbalancer/base.py | 12 ++++++------ libcloud/loadbalancer/drivers/gogrid.py | 22 +++++++++++----------- libcloud/loadbalancer/drivers/rackspace.py | 16 ++++++++-------- libcloud/loadbalancer/types.py | 4 ++-- test/loadbalancer/test_gogrid.py | 14 +++++++------- test/loadbalancer/test_rackspace.py | 8 ++++---- 7 files changed, 44 insertions(+), 44 deletions(-) diff --git a/example_loadbalancer.py b/example_loadbalancer.py index 10dc6e72fc..f7bc783386 100644 --- a/example_loadbalancer.py +++ b/example_loadbalancer.py @@ -3,8 +3,8 @@ import os import time -from libcloud.loadbalancer.base import LB, LBMember -from libcloud.loadbalancer.types import Provider, LBState +from libcloud.loadbalancer.base import LoadBalancer, Member +from libcloud.loadbalancer.types import Provider, State from libcloud.loadbalancer.providers import get_driver def main(): @@ -20,8 +20,8 @@ def main(): new_balancer_name = 'testlb' + os.urandom(4).encode('hex') new_balancer = driver.create_balancer(name=new_balancer_name, port=80, - nodes=(LBMember(None, '192.168.86.1', 80), - LBMember(None, '192.168.86.2', 8080)) + nodes=(Member(None, '192.168.86.1', 80), + Member(None, '192.168.86.2', 8080)) ) print new_balancer @@ -32,7 +32,7 @@ def main(): while True: balancer = driver.balancer_detail(balancer=new_balancer) - if balancer.state == LBState.RUNNING: + if balancer.state == State.RUNNING: break time.sleep(30) @@ -45,7 +45,7 @@ def main(): balancer.detach_node(nodes[0]) # and add another one: 10.0.0.10:1000 - print balancer.attach_node(ip='10.0.0.10', port='1000') + print balancer.attach_node(Member(None, ip='10.0.0.10', port='1000')) # remove the balancer driver.destroy_balancer(new_balancer) diff --git a/libcloud/loadbalancer/base.py b/libcloud/loadbalancer/base.py index 7c60692f97..55fd3f4505 100644 --- a/libcloud/loadbalancer/base.py +++ b/libcloud/loadbalancer/base.py @@ -17,14 +17,14 @@ from libcloud.common.types import LibcloudError __all__ = [ - "LBMember", + "Member", "LB", "LBDriver", "LBAlgorithm", "LBProtocol" ] -class LBMember(object): +class Member(object): def __init__(self, id, ip, port): self.id = str(id) if id else None @@ -35,14 +35,14 @@ def __repr__(self): return ('' % (self.id, self.ip, self.port)) -class LBAlgorithm(object): +class Algorithm(object): RANDOM = 0 ROUND_ROBIN = 1 LEAST_CONNECTIONS = 2 -DEFAULT_ALGORITHM = LBAlgorithm.ROUND_ROBIN +DEFAULT_ALGORITHM = Algorithm.ROUND_ROBIN -class LB(object): +class LoadBalancer(object): """ Provide a common interface for handling Load Balancers. """ @@ -72,7 +72,7 @@ def __repr__(self): self.name, self.state)) -class LBDriver(object): +class Driver(object): """ A base LBDriver class to derive from diff --git a/libcloud/loadbalancer/drivers/gogrid.py b/libcloud/loadbalancer/drivers/gogrid.py index 224165d34a..d592055367 100644 --- a/libcloud/loadbalancer/drivers/gogrid.py +++ b/libcloud/loadbalancer/drivers/gogrid.py @@ -18,22 +18,22 @@ from libcloud.common.types import LibcloudError from libcloud.utils import reverse_dict from libcloud.common.gogrid import GoGridConnection, BaseGoGridDriver -from libcloud.loadbalancer.base import LB, LBMember, LBDriver, LBAlgorithm +from libcloud.loadbalancer.base import LoadBalancer, Member, Driver, Algorithm from libcloud.loadbalancer.base import DEFAULT_ALGORITHM -from libcloud.loadbalancer.types import Provider, LBState, LibcloudLBImmutableError +from libcloud.loadbalancer.types import Provider, State, LibcloudLBImmutableError -class GoGridLBDriver(BaseGoGridDriver, LBDriver): +class GoGridLBDriver(BaseGoGridDriver, Driver): connectionCls = GoGridConnection type = Provider.RACKSPACE api_name = 'gogrid_lb' name = 'GoGrid LB' - LB_STATE_MAP = { 'On': LBState.RUNNING, - 'Unknown': LBState.UNKNOWN } + LB_STATE_MAP = { 'On': State.RUNNING, + 'Unknown': State.UNKNOWN } _VALUE_TO_ALGORITHM_MAP = { - 'round balancer': LBAlgorithm.ROUND_ROBIN, - 'least connection': LBAlgorithm.LEAST_CONNECTIONS + 'round balancer': Algorithm.ROUND_ROBIN, + 'least connection': Algorithm.LEAST_CONNECTIONS } _ALGORITHM_TO_VALUE_MAP = reverse_dict(_VALUE_TO_ALGORITHM_MAP) @@ -151,7 +151,7 @@ def _update_balancer(self, params): def _members_to_params(self, members): """ - Helper method to convert list of L{LBMember} objects + Helper method to convert list of L{Member} objects to GET params. """ @@ -170,10 +170,10 @@ def _to_balancers(self, object): return [ self._to_balancer(el) for el in object["list"] ] def _to_balancer(self, el): - lb = LB(id=el.get("id"), + lb = LoadBalancer(id=el.get("id"), name=el["name"], state=self.LB_STATE_MAP.get( - el["state"]["name"], LBState.UNKNOWN), + el["state"]["name"], State.UNKNOWN), ip=el["virtualip"]["ip"]["ip"], port=el["virtualip"]["port"], driver=self.connection.driver) @@ -183,7 +183,7 @@ def _to_members(self, object): return [ self._to_member(el) for el in object ] def _to_member(self, el): - member = LBMember(id=el["ip"]["id"], + member = Member(id=el["ip"]["id"], ip=el["ip"]["ip"], port=el["port"]) return member diff --git a/libcloud/loadbalancer/drivers/rackspace.py b/libcloud/loadbalancer/drivers/rackspace.py index 610d4203ed..666053faad 100644 --- a/libcloud/loadbalancer/drivers/rackspace.py +++ b/libcloud/loadbalancer/drivers/rackspace.py @@ -21,8 +21,8 @@ import simplejson as json from libcloud.common.base import Response -from libcloud.loadbalancer.base import LB, LBMember, LBDriver -from libcloud.loadbalancer.types import Provider, LBState +from libcloud.loadbalancer.base import LoadBalancer, Member, Driver +from libcloud.loadbalancer.types import Provider, State from libcloud.common.rackspace import (AUTH_HOST_US, RackspaceBaseConnection) @@ -63,14 +63,14 @@ def request(self, action, params=None, data='', headers=None, method='GET'): params=params, data=data, method=method, headers=headers) -class RackspaceLBDriver(LBDriver): +class RackspaceLBDriver(Driver): connectionCls = RackspaceConnection type = Provider.RACKSPACE api_name = 'rackspace_lb' name = 'Rackspace LB' - LB_STATE_MAP = { 'ACTIVE': LBState.RUNNING, - 'BUILD': LBState.PENDING } + LB_STATE_MAP = { 'ACTIVE': State.RUNNING, + 'BUILD': State.PENDING } def list_balancers(self): return self._to_balancers( @@ -139,10 +139,10 @@ def _to_balancers(self, object): return [ self._to_balancer(el) for el in object["loadBalancers"] ] def _to_balancer(self, el): - lb = LB(id=el["id"], + lb = LoadBalancer(id=el["id"], name=el["name"], state=self.LB_STATE_MAP.get( - el["status"], LBState.UNKNOWN), + el["status"], State.UNKNOWN), ip=el["virtualIps"][0]["address"], port=el["port"], driver=self.connection.driver) @@ -152,7 +152,7 @@ def _to_members(self, object): return [ self._to_member(el) for el in object["nodes"] ] def _to_member(self, el): - lbmember = LBMember(id=el["id"], + lbmember = Member(id=el["id"], ip=el["address"], port=el["port"]) return lbmember diff --git a/libcloud/loadbalancer/types.py b/libcloud/loadbalancer/types.py index 143ca6f802..4f84c39ef0 100644 --- a/libcloud/loadbalancer/types.py +++ b/libcloud/loadbalancer/types.py @@ -15,7 +15,7 @@ __all__ = [ "Provider", - "LBState", + "State", "LibcloudLBError", "LibcloudLBImmutableError", ] @@ -30,7 +30,7 @@ class Provider(object): RACKSPACE = 0 GOGRID = 1 -class LBState(object): +class State(object): """ Standart states for a loadbalancer diff --git a/test/loadbalancer/test_gogrid.py b/test/loadbalancer/test_gogrid.py index f7cc557e82..f15474deeb 100644 --- a/test/loadbalancer/test_gogrid.py +++ b/test/loadbalancer/test_gogrid.py @@ -3,7 +3,7 @@ import sys import unittest -from libcloud.loadbalancer.base import LB, LBMember, LBAlgorithm +from libcloud.loadbalancer.base import LoadBalancer, Member, Algorithm from libcloud.loadbalancer.drivers.gogrid import GoGridLBDriver from test import MockHttp, MockRawResponse @@ -29,9 +29,9 @@ def test_list_balancers(self): def test_create_balancer(self): balancer = self.driver.create_balancer(name='test2', port=80, - algorithm=LBAlgorithm.ROUND_ROBIN, - members=(LBMember(None, '10.1.0.10', 80), - LBMember(None, '10.1.0.11', 80)) + algorithm=Algorithm.ROUND_ROBIN, + members=(Member(None, '10.1.0.10', 80), + Member(None, '10.1.0.11', 80)) ) self.assertEquals(balancer.name, 'test2') @@ -61,15 +61,15 @@ def test_balancer_list_members(self): set(["%s:%s" % (member.ip, member.port) for member in members])) def test_balancer_attach_member(self): - balancer = LB(23530, None, None, None, None, None) + balancer = LoadBalancer(23530, None, None, None, None, None) member = self.driver.balancer_attach_member(balancer, - LBMember(None, ip='10.0.0.75', port='80')) + Member(None, ip='10.0.0.75', port='80')) self.assertEquals(member.ip, '10.0.0.75') self.assertEquals(member.port, 80) def test_balancer_detach_member(self): - balancer = LB(23530, None, None, None, None, None) + balancer = LoadBalancer(23530, None, None, None, None, None) member = self.driver.balancer_list_members(balancer)[0] ret = self.driver.balancer_detach_member(balancer, member) diff --git a/test/loadbalancer/test_rackspace.py b/test/loadbalancer/test_rackspace.py index cf3d65d93e..34e6371e7b 100644 --- a/test/loadbalancer/test_rackspace.py +++ b/test/loadbalancer/test_rackspace.py @@ -3,7 +3,7 @@ import sys import unittest -from libcloud.loadbalancer.base import LB, LBMember +from libcloud.loadbalancer.base import Member from libcloud.loadbalancer.drivers.rackspace import RackspaceLBDriver from test import MockHttp, MockRawResponse @@ -29,8 +29,8 @@ def test_list_balancers(self): def test_create_balancer(self): balancer = self.driver.create_balancer(name='test2', port=80, - members=(LBMember(None, '10.1.0.10', 80), - LBMember(None, '10.1.0.11', 80)) + members=(Member(None, '10.1.0.10', 80), + Member(None, '10.1.0.11', 80)) ) self.assertEquals(balancer.name, 'test2') @@ -58,7 +58,7 @@ def test_balancer_list_members(self): def test_balancer_attach_member(self): balancer = self.driver.get_balancer(balancer_id='8290') - member = balancer.attach_member(LBMember(None, ip='10.1.0.12', port='80')) + member = balancer.attach_member(Member(None, ip='10.1.0.12', port='80')) self.assertEquals(member.ip, '10.1.0.12') self.assertEquals(member.port, 80) From d99132fe126375f80a115d37b2215f3089706311 Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Sat, 14 May 2011 13:25:31 +0000 Subject: [PATCH 31/90] remove unused imports git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103101 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/compute/drivers/opsource.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/libcloud/compute/drivers/opsource.py b/libcloud/compute/drivers/opsource.py index ad9f48f8c6..6d39b3b572 100644 --- a/libcloud/compute/drivers/opsource.py +++ b/libcloud/compute/drivers/opsource.py @@ -16,9 +16,7 @@ Opsource Driver """ import base64 -import socket from xml.etree import ElementTree as ET -from xml.parsers.expat import ExpatError from libcloud.utils import fixxpath, findtext, findall from libcloud.common.base import ConnectionUserAndKey, Response From 517484014d1ef353751ca14bc59e7da2ad6de1dc Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Sat, 14 May 2011 13:26:56 +0000 Subject: [PATCH 32/90] fix __all__ for load balancers git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103103 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/base.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/libcloud/loadbalancer/base.py b/libcloud/loadbalancer/base.py index 55fd3f4505..03523ad42d 100644 --- a/libcloud/loadbalancer/base.py +++ b/libcloud/loadbalancer/base.py @@ -18,10 +18,9 @@ __all__ = [ "Member", - "LB", - "LBDriver", - "LBAlgorithm", - "LBProtocol" + "LoadBalancer", + "Driver", + "Algorithm" ] class Member(object): From eb48f8ac003207431ae373f9ed3f444559de0a88 Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Sat, 14 May 2011 13:27:55 +0000 Subject: [PATCH 33/90] Ooops, used old name here git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103104 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcloud/loadbalancer/base.py b/libcloud/loadbalancer/base.py index 03523ad42d..c75e065c92 100644 --- a/libcloud/loadbalancer/base.py +++ b/libcloud/loadbalancer/base.py @@ -169,7 +169,7 @@ def balancer_attach_compute_node(self, balancer, node): @return {LBMember} Member after joining the balancer. """ - return self.attach_member(LBMember(None, node.public_ip[0], balancer.port)) + return self.attach_member(Member(None, node.public_ip[0], balancer.port)) def balancer_attach_member(self, balancer, member): """ From c17ec3c53ff808af29c61929cfc7d427bd76060a Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Sat, 14 May 2011 13:28:07 +0000 Subject: [PATCH 34/90] Remove unused imports git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103105 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/compute/drivers/gogrid.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/libcloud/compute/drivers/gogrid.py b/libcloud/compute/drivers/gogrid.py index 6f1986bd33..27c9f7c2f2 100644 --- a/libcloud/compute/drivers/gogrid.py +++ b/libcloud/compute/drivers/gogrid.py @@ -19,11 +19,6 @@ import hashlib import copy -try: - import json -except ImportError: - import simplejson as json - from libcloud.common.types import InvalidCredsError, LibcloudError from libcloud.common.gogrid import GoGridConnection, BaseGoGridDriver from libcloud.compute.providers import Provider From 25c4befd6006a7a203fa347e3620eb0368ad3428 Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Sat, 14 May 2011 13:30:22 +0000 Subject: [PATCH 35/90] remove unused import git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103108 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/compute/drivers/dreamhost.py | 1 - 1 file changed, 1 deletion(-) diff --git a/libcloud/compute/drivers/dreamhost.py b/libcloud/compute/drivers/dreamhost.py index 59d03f8558..fb857b640a 100644 --- a/libcloud/compute/drivers/dreamhost.py +++ b/libcloud/compute/drivers/dreamhost.py @@ -23,7 +23,6 @@ import copy -from libcloud.pricing import get_pricing from libcloud.common.base import ConnectionKey, Response from libcloud.common.types import InvalidCredsError from libcloud.compute.base import Node, NodeDriver, NodeSize From 1f048da4d38eb50f7cb1117ef552374eeab900be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 14 May 2011 13:31:38 +0000 Subject: [PATCH 36/90] Update create_balancer method in the GoGrid driver. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103109 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/drivers/gogrid.py | 20 ++++++++++++-------- libcloud/loadbalancer/drivers/rackspace.py | 19 ++++++++++++++----- test/loadbalancer/test_gogrid.py | 9 ++++++++- test/loadbalancer/test_rackspace.py | 3 ++- 4 files changed, 36 insertions(+), 15 deletions(-) diff --git a/libcloud/loadbalancer/drivers/gogrid.py b/libcloud/loadbalancer/drivers/gogrid.py index d592055367..6be2193b38 100644 --- a/libcloud/loadbalancer/drivers/gogrid.py +++ b/libcloud/loadbalancer/drivers/gogrid.py @@ -19,7 +19,7 @@ from libcloud.utils import reverse_dict from libcloud.common.gogrid import GoGridConnection, BaseGoGridDriver from libcloud.loadbalancer.base import LoadBalancer, Member, Driver, Algorithm -from libcloud.loadbalancer.base import DEFAULT_ALGORITHM +from libcloud.loadbalancer.base import DEFAULT_ALGORITHM from libcloud.loadbalancer.types import Provider, State, LibcloudLBImmutableError @@ -37,15 +37,17 @@ class GoGridLBDriver(BaseGoGridDriver, Driver): } _ALGORITHM_TO_VALUE_MAP = reverse_dict(_VALUE_TO_ALGORITHM_MAP) + def list_protocols(self): + # GoGrid only supports http + return [ 'http' ] + def list_balancers(self): return self._to_balancers( self.connection.request('/api/grid/loadbalancer/list').object) - def ex_create_balancer_nowait(self, name, port, algorithm, members): - if not algorithm: - algorithm = DEFAULT_ALGORITHM - else: - algorithm = self._algorithm_to_value(algorithm) + def ex_create_balancer_nowait(self, name, members, protocol='http', port=80, + algorithm=Algorithm.ROUND_ROBIN): + algorithm = self._algorithm_to_value(algorithm) params = {'name': name, 'loadbalancer.type': algorithm, @@ -58,8 +60,10 @@ def ex_create_balancer_nowait(self, name, port, algorithm, members): params=params) return self._to_balancers(resp.object)[0] - def create_balancer(self, name, port, algorithm, members): - balancer = self.ex_create_balancer_nowait(name, port, algorithm, members) + def create_balancer(self, name, members, protocol='http', port=80, + algorithm=Algorithm.ROUND_ROBIN): + balancer = self.ex_create_balancer_nowait(name, members, protocol, + port, algorithm) timeout = 60 * 20 waittime = 0 diff --git a/libcloud/loadbalancer/drivers/rackspace.py b/libcloud/loadbalancer/drivers/rackspace.py index 666053faad..9b0d1ece61 100644 --- a/libcloud/loadbalancer/drivers/rackspace.py +++ b/libcloud/loadbalancer/drivers/rackspace.py @@ -20,8 +20,9 @@ except ImportError: import simplejson as json +from libcloud.utils import reverse_dict from libcloud.common.base import Response -from libcloud.loadbalancer.base import LoadBalancer, Member, Driver +from libcloud.loadbalancer.base import LoadBalancer, Member, Driver, Algorithm from libcloud.loadbalancer.types import Provider, State from libcloud.common.rackspace import (AUTH_HOST_US, RackspaceBaseConnection) @@ -71,19 +72,27 @@ class RackspaceLBDriver(Driver): LB_STATE_MAP = { 'ACTIVE': State.RUNNING, 'BUILD': State.PENDING } + _VALUE_TO_ALGORITHM_MAP = { + 'RANDOM': Algorithm.RANDOM, + 'ROUND_ROBIN': Algorithm.ROUND_ROBIN, + 'LEAST_CONNECTIONS': Algorithm.LEAST_CONNECTIONS + } + _ALGORITHM_TO_VALUE_MAP = reverse_dict(_VALUE_TO_ALGORITHM_MAP) def list_balancers(self): return self._to_balancers( self.connection.request('/loadbalancers').object) - def create_balancer(self, **kwargs): - name = kwargs['name'] - port = kwargs['port'] - members = kwargs['members'] + def create_balancer(self, name, port, algorithm, members): + if not algorithm: + algorithm = DEFAULT_ALGORITHM + else: + algorithm = self._algorithm_to_value(algorithm) balancer_object = {"loadBalancer": {"name": name, "port": port, + "algorithm": algorithm, "protocol": "HTTP", "virtualIps": [{"type": "PUBLIC"}], "nodes": [{"address": member.ip, diff --git a/test/loadbalancer/test_gogrid.py b/test/loadbalancer/test_gogrid.py index f15474deeb..9d5b422b25 100644 --- a/test/loadbalancer/test_gogrid.py +++ b/test/loadbalancer/test_gogrid.py @@ -17,6 +17,12 @@ def setUp(self): GoGridLBMockHttp.type = None self.driver = GoGridLBDriver('user', 'key') + def test_list_protocols(self): + protocols = self.driver.list_protocols() + + self.assertEqual(len(protocols), 1) + self.assertEqual(protocols[0], 'http') + def test_list_balancers(self): balancers = self.driver.list_balancers() @@ -29,9 +35,10 @@ def test_list_balancers(self): def test_create_balancer(self): balancer = self.driver.create_balancer(name='test2', port=80, + protocol='http', algorithm=Algorithm.ROUND_ROBIN, members=(Member(None, '10.1.0.10', 80), - Member(None, '10.1.0.11', 80)) + Member(None, '10.1.0.11', 80)) ) self.assertEquals(balancer.name, 'test2') diff --git a/test/loadbalancer/test_rackspace.py b/test/loadbalancer/test_rackspace.py index 34e6371e7b..c6c6995f42 100644 --- a/test/loadbalancer/test_rackspace.py +++ b/test/loadbalancer/test_rackspace.py @@ -3,7 +3,7 @@ import sys import unittest -from libcloud.loadbalancer.base import Member +from libcloud.loadbalancer.base import Member, Algorithm from libcloud.loadbalancer.drivers.rackspace import RackspaceLBDriver from test import MockHttp, MockRawResponse @@ -29,6 +29,7 @@ def test_list_balancers(self): def test_create_balancer(self): balancer = self.driver.create_balancer(name='test2', port=80, + algorithm=Algorithm.ROUND_ROBIN, members=(Member(None, '10.1.0.10', 80), Member(None, '10.1.0.11', 80)) ) From d9fd09cb6c31ca59e18ee4fa824d5b4036a664a6 Mon Sep 17 00:00:00 2001 From: Paul Querna Date: Sat, 14 May 2011 13:31:41 +0000 Subject: [PATCH 37/90] remove more unused imports (padding my stats yo) git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103110 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/compute/drivers/vpsnet.py | 2 -- libcloud/drivers/cloudsigma.py | 1 - 2 files changed, 3 deletions(-) diff --git a/libcloud/compute/drivers/vpsnet.py b/libcloud/compute/drivers/vpsnet.py index edd322ab47..96664284a2 100644 --- a/libcloud/compute/drivers/vpsnet.py +++ b/libcloud/compute/drivers/vpsnet.py @@ -22,8 +22,6 @@ except: import simplejson as json -from libcloud.pricing import get_pricing - from libcloud.common.base import ConnectionUserAndKey, Response from libcloud.common.types import InvalidCredsError from libcloud.compute.providers import Provider diff --git a/libcloud/drivers/cloudsigma.py b/libcloud/drivers/cloudsigma.py index bf09de855e..f4587fc32d 100644 --- a/libcloud/drivers/cloudsigma.py +++ b/libcloud/drivers/cloudsigma.py @@ -16,7 +16,6 @@ from libcloud.utils import deprecated_warning -from libcloud.utils import str2dicts, str2list, dict2str from libcloud.compute.drivers.cloudsigma import * deprecated_warning(__name__) From 90d35872e1dbcae9d4e3cc5d5c5bb1f9c92ff423 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 14 May 2011 13:48:44 +0000 Subject: [PATCH 38/90] Update the create_balancer method in rackspace driver. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103111 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/drivers/gogrid.py | 2 +- libcloud/loadbalancer/drivers/rackspace.py | 11 ++++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/libcloud/loadbalancer/drivers/gogrid.py b/libcloud/loadbalancer/drivers/gogrid.py index 6be2193b38..35f05e7640 100644 --- a/libcloud/loadbalancer/drivers/gogrid.py +++ b/libcloud/loadbalancer/drivers/gogrid.py @@ -46,7 +46,7 @@ def list_balancers(self): self.connection.request('/api/grid/loadbalancer/list').object) def ex_create_balancer_nowait(self, name, members, protocol='http', port=80, - algorithm=Algorithm.ROUND_ROBIN): + algorithm=DEFAULT_ALGORITHM): algorithm = self._algorithm_to_value(algorithm) params = {'name': name, diff --git a/libcloud/loadbalancer/drivers/rackspace.py b/libcloud/loadbalancer/drivers/rackspace.py index 9b0d1ece61..aaadce55d3 100644 --- a/libcloud/loadbalancer/drivers/rackspace.py +++ b/libcloud/loadbalancer/drivers/rackspace.py @@ -23,6 +23,7 @@ from libcloud.utils import reverse_dict from libcloud.common.base import Response from libcloud.loadbalancer.base import LoadBalancer, Member, Driver, Algorithm +from libcloud.loadbalancer.base import DEFAULT_ALGORITHM from libcloud.loadbalancer.types import Provider, State from libcloud.common.rackspace import (AUTH_HOST_US, RackspaceBaseConnection) @@ -83,17 +84,13 @@ def list_balancers(self): return self._to_balancers( self.connection.request('/loadbalancers').object) - def create_balancer(self, name, port, algorithm, members): - if not algorithm: - algorithm = DEFAULT_ALGORITHM - else: - algorithm = self._algorithm_to_value(algorithm) - + def create_balancer(self, name, members, protocol='http', + port=80, algorithm=DEFAULT_ALGORITHM): balancer_object = {"loadBalancer": {"name": name, "port": port, "algorithm": algorithm, - "protocol": "HTTP", + "protocol": protocol.upper(), "virtualIps": [{"type": "PUBLIC"}], "nodes": [{"address": member.ip, "port": member.port, From f8f97a6867b001c79b12b722182824c6682eba21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 14 May 2011 13:55:07 +0000 Subject: [PATCH 39/90] Use DEFAULT_ALGORITHM constant. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103113 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/drivers/gogrid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcloud/loadbalancer/drivers/gogrid.py b/libcloud/loadbalancer/drivers/gogrid.py index 35f05e7640..dd7b6db952 100644 --- a/libcloud/loadbalancer/drivers/gogrid.py +++ b/libcloud/loadbalancer/drivers/gogrid.py @@ -61,7 +61,7 @@ def ex_create_balancer_nowait(self, name, members, protocol='http', port=80, return self._to_balancers(resp.object)[0] def create_balancer(self, name, members, protocol='http', port=80, - algorithm=Algorithm.ROUND_ROBIN): + algorithm=DEFAULT_ALGORITHM): balancer = self.ex_create_balancer_nowait(name, members, protocol, port, algorithm) From 86b4557f3c81ee3a51a1c693e02dfda4260bbd8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 14 May 2011 15:26:51 +0000 Subject: [PATCH 40/90] Add list_protocols method to the Rackspace LB driver. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103136 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/drivers/rackspace.py | 10 ++++++++++ test/loadbalancer/test_rackspace.py | 11 +++++++++++ 2 files changed, 21 insertions(+) diff --git a/libcloud/loadbalancer/drivers/rackspace.py b/libcloud/loadbalancer/drivers/rackspace.py index aaadce55d3..40bc204994 100644 --- a/libcloud/loadbalancer/drivers/rackspace.py +++ b/libcloud/loadbalancer/drivers/rackspace.py @@ -80,6 +80,10 @@ class RackspaceLBDriver(Driver): } _ALGORITHM_TO_VALUE_MAP = reverse_dict(_VALUE_TO_ALGORITHM_MAP) + def list_protocols(self): + return self._to_protocols( + self.connection.request('/loadbalancers/protocols').object) + def list_balancers(self): return self._to_balancers( self.connection.request('/loadbalancers').object) @@ -141,6 +145,12 @@ def balancer_list_members(self, balancer): return self._to_members( self.connection.request(uri).object) + def _to_protocols(self, object): + protocols = [] + for item in object["protocols"]: + protocols.append(item['name'].lower()) + return protocols + def _to_balancers(self, object): return [ self._to_balancer(el) for el in object["loadBalancers"] ] diff --git a/test/loadbalancer/test_rackspace.py b/test/loadbalancer/test_rackspace.py index c6c6995f42..297bc7f093 100644 --- a/test/loadbalancer/test_rackspace.py +++ b/test/loadbalancer/test_rackspace.py @@ -17,6 +17,12 @@ def setUp(self): RackspaceLBMockHttp.type = None self.driver = RackspaceLBDriver('user', 'key') + def test_list_protocols(self): + protocols = self.driver.list_protocols() + + self.assertEqual(len(protocols), 10) + self.assertTrue('http' in protocols) + def test_list_balancers(self): balancers = self.driver.list_balancers() @@ -83,6 +89,11 @@ def _v1_0(self, method, url, body, headers): 'x-storage-url': 'https://storage4.clouddrive.com/v1/MossoCloudFS_FE011C19-CF86-4F87-BE5D-9229145D7A06'} return (httplib.NO_CONTENT, "", headers, httplib.responses[httplib.NO_CONTENT]) + def _v1_0_slug_loadbalancers_protocols(self, method, url, body, headers): + body = self.fixtures.load('v1_slug_loadbalancers_protocols.json') + return (httplib.ACCEPTED, body, {}, + httplib.responses[httplib.ACCEPTED]) + def _v1_0_slug_loadbalancers(self, method, url, body, headers): if method == "GET": body = self.fixtures.load('v1_slug_loadbalancers.json') From ce988967ed9f17e432dbab1ca28527faf595ec80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 14 May 2011 16:14:22 +0000 Subject: [PATCH 41/90] Forgot to add this file. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103149 13f79535-47bb-0310-9956-ffa450edef68 --- .../v1_slug_loadbalancers_protocols.json | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 test/loadbalancer/fixtures/rackspace/v1_slug_loadbalancers_protocols.json diff --git a/test/loadbalancer/fixtures/rackspace/v1_slug_loadbalancers_protocols.json b/test/loadbalancer/fixtures/rackspace/v1_slug_loadbalancers_protocols.json new file mode 100644 index 0000000000..d966c396b7 --- /dev/null +++ b/test/loadbalancer/fixtures/rackspace/v1_slug_loadbalancers_protocols.json @@ -0,0 +1,43 @@ +{"protocols": [ + { + "name": "HTTP", + "port": "80" + }, + { + "name": "FTP", + "port": "21" + }, + { + "name": "IMAPv4", + "port": "143" + }, + { + "name": "POP3", + "port": "110" + }, + { + "name": "SMTP", + "port": "25" + }, + { + "name": "LDAP", + "port": "389" + }, + { + "name": "HTTPS", + "port": "443" + }, + { + "name": "IMAPS", + "port": "993" + }, + { + "name": "POP3S", + "port": "995" + }, + { + "name": "LDAPS", + "port": "636" + } + ] +} From 824d738e60b2797110d84f51f6cedee11ae1d2fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 14 May 2011 23:23:11 +0000 Subject: [PATCH 42/90] Remove file_hash and add verify_hash argument and make it consistent across all the drivers and update affects tests. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103251 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/storage/base.py | 12 +++++------- libcloud/storage/drivers/cloudfiles.py | 22 ++++++++++++---------- libcloud/storage/drivers/dummy.py | 6 ++++++ libcloud/storage/drivers/s3.py | 18 +++++++++--------- libcloud/storage/types.py | 4 ++-- test/storage/test_cloudfiles.py | 20 ++++++++++++++------ test/storage/test_s3.py | 12 +++++++----- 7 files changed, 55 insertions(+), 39 deletions(-) diff --git a/libcloud/storage/base.py b/libcloud/storage/base.py index 666f6a1e2a..a66db7f562 100644 --- a/libcloud/storage/base.py +++ b/libcloud/storage/base.py @@ -122,9 +122,9 @@ def get_object(self, object_name): return self.driver.get_object(container_name=self.name, object_name=object_name) - def upload_object(self, file_path, object_name, extra=None, file_hash=None): + def upload_object(self, file_path, object_name, extra=None, verify_hash=True): return self.driver.upload_object( - file_path, self, object_name, extra, file_hash) + file_path, self, object_name, extra, verify_hash) def upload_object_via_stream(self, iterator, object_name, extra=None): return self.driver.upload_object_via_stream( @@ -301,7 +301,7 @@ def download_object_as_stream(self, obj, chunk_size=None): 'download_object_as_stream not implemented for this driver') def upload_object(self, file_path, container, object_name, extra=None, - file_hash=None): + verify_hash=True): """ Upload an object. @@ -317,10 +317,8 @@ def upload_object(self, file_path, container, object_name, extra=None, @type extra: C{dict} @param extra: (optional) Extra attributes (driver specific). - @type file_hash: C{str} - @param file_hash: (optional) File hash. If provided object hash is - on upload and if it doesn't match the one provided an - exception is thrown. + @type verify_hash: C{boolean} + @param verify_hash: True to do a file integrity check. """ raise NotImplementedError( 'upload_object not implemented for this driver') diff --git a/libcloud/storage/drivers/cloudfiles.py b/libcloud/storage/drivers/cloudfiles.py index 6783ffc6c2..c3e70f69a5 100644 --- a/libcloud/storage/drivers/cloudfiles.py +++ b/libcloud/storage/drivers/cloudfiles.py @@ -203,7 +203,6 @@ def get_object(self, container_name, object_name): response = self.connection.request('/%s/%s' % (container_name, object_name), method='HEAD') - if response.status in [ httplib.OK, httplib.NO_CONTENT ]: obj = self._headers_to_object( object_name, container, response.headers) @@ -309,7 +308,7 @@ def download_object_as_stream(self, obj, chunk_size=None): success_status_code=httplib.OK) def upload_object(self, file_path, container, object_name, extra=None, - file_hash=None): + verify_hash=True): """ Upload an object. @@ -322,7 +321,7 @@ def upload_object(self, file_path, container, object_name, extra=None, upload_func=upload_func, upload_func_kwargs=upload_func_kwargs, extra=extra, file_path=file_path, - file_hash=file_hash) + verify_hash=verify_hash) def upload_object_via_stream(self, iterator, container, object_name, extra=None): @@ -354,7 +353,7 @@ def delete_object(self, obj): def _put_object(self, container, object_name, upload_func, upload_func_kwargs, extra=None, file_path=None, - iterator=None, file_hash=None): + iterator=None, verify_hash=True): extra = extra or {} container_name_cleaned = self._clean_container_name(container.name) object_name_cleaned = self._clean_object_name(object_name) @@ -362,9 +361,6 @@ def _put_object(self, container, object_name, upload_func, meta_data = extra.get('meta_data', None) headers = {} - if not iterator and file_hash: - headers['ETag'] = file_hash - if meta_data: for key, value in meta_data.iteritems(): key = 'X-Object-Meta-%s' % (key) @@ -382,17 +378,23 @@ def _put_object(self, container, object_name, upload_func, response = result_dict['response'].response bytes_transferred = result_dict['bytes_transferred'] + print result_dict['data_hash'] + server_hash = result_dict['response'].headers.get('etag', None) if response.status == httplib.EXPECTATION_FAILED: raise LibcloudError(value='Missing content-type header', driver=self) - elif response.status == httplib.UNPROCESSABLE_ENTITY: + elif verify_hash and not server_hash: + raise LibcloudError(value='Server didn\'t return etag', + driver=self) + elif (verify_hash and result_dict['data_hash'] != server_hash): raise ObjectHashMismatchError( - value='MD5 hash checksum does not match', + value=('MD5 hash checksum does not match (expected=%s, ' + + 'actual=%s)') % (result_dict['data_hash'], server_hash), object_name=object_name, driver=self) elif response.status == httplib.CREATED: obj = Object( - name=object_name, size=bytes_transferred, hash=file_hash, + name=object_name, size=bytes_transferred, hash=server_hash, extra=None, meta_data=meta_data, container=container, driver=self) diff --git a/libcloud/storage/drivers/dummy.py b/libcloud/storage/drivers/dummy.py index 058e42f8b3..274225d0b9 100644 --- a/libcloud/storage/drivers/dummy.py +++ b/libcloud/storage/drivers/dummy.py @@ -15,6 +15,7 @@ import os.path import random +import hashlib from libcloud.common.types import LibcloudError @@ -48,14 +49,19 @@ def __len__(self): class DummyIterator(object): def __init__(self, data=None): + self.hash = hashlib.md5() self._data = data or [] self._current_item = 0 + def get_md5_hash(self): + return self.hash.hexdigest() + def next(self): if self._current_item == len(self._data): raise StopIteration value = self._data[self._current_item] + self.hash.update(value) self._current_item += 1 return value diff --git a/libcloud/storage/drivers/s3.py b/libcloud/storage/drivers/s3.py index 9ee075a647..75bef17242 100644 --- a/libcloud/storage/drivers/s3.py +++ b/libcloud/storage/drivers/s3.py @@ -290,7 +290,7 @@ def download_object_as_stream(self, obj, chunk_size=None): success_status_code=httplib.OK) def upload_object(self, file_path, container, object_name, extra=None, - file_hash=None, ex_storage_class=None): + verify_hash=True, ex_storage_class=None): upload_func = self._upload_file upload_func_kwargs = { 'file_path': file_path } @@ -298,7 +298,7 @@ def upload_object(self, file_path, container, object_name, extra=None, upload_func=upload_func, upload_func_kwargs=upload_func_kwargs, extra=extra, file_path=file_path, - file_hash=file_hash, + verify_hash=verify_hash, storage_class=ex_storage_class) def upload_object_via_stream(self, iterator, container, object_name, @@ -328,7 +328,7 @@ def _clean_object_name(self, name): def _put_object(self, container, object_name, upload_func, upload_func_kwargs, extra=None, file_path=None, - iterator=None, file_hash=None, storage_class=None): + iterator=None, verify_hash=True, storage_class=None): headers = {} extra = extra or {} storage_class = storage_class or 'standard' @@ -342,9 +342,6 @@ def _put_object(self, container, object_name, upload_func, content_type = extra.get('content_type', None) meta_data = extra.get('meta_data', None) - if not iterator and file_hash: - headers['Content-MD5'] = base64.b64encode(file_hash.decode('hex')) - if meta_data: for key, value in meta_data.iteritems(): key = 'x-amz-meta-%s' % (key) @@ -368,19 +365,22 @@ def _put_object(self, container, object_name, upload_func, bytes_transferred = result_dict['bytes_transferred'] headers = response.headers response = response.response + server_hash = headers['etag'].replace('"', '') - if (file_hash and response.status == httplib.BAD_REQUEST) or \ - (file_hash and file_hash != headers['etag'].replace('"', '')): + if (verify_hash and result_dict['data_hash'] != server_hash): raise ObjectHashMismatchError( value='MD5 hash checksum does not match', object_name=object_name, driver=self) elif response.status == httplib.OK: obj = Object( - name=object_name, size=bytes_transferred, hash=file_hash, + name=object_name, size=bytes_transferred, hash=server_hash, extra=None, meta_data=meta_data, container=container, driver=self) return obj + else: + raise LibcloudError('Unexpected status code, status_code=%s' % (response.status), + driver=self) def _to_containers(self, obj, xpath): return [ self._to_container(element) for element in \ diff --git a/libcloud/storage/types.py b/libcloud/storage/types.py index 8aa1dc0df5..195fb40acf 100644 --- a/libcloud/storage/types.py +++ b/libcloud/storage/types.py @@ -67,8 +67,8 @@ def __init__(self, value, driver, object_name): super(ObjectError, self).__init__(value=value, driver=driver) def __str__(self): - return '<%s in %s, object = %s>' % (self.error_type, repr(self.driver), - self.object_name) + return '<%s in %s, value=%s, object = %s>' % (self.error_type, repr(self.driver), + self.value, self.object_name) class ContainerAlreadyExistsError(ContainerError): error_type = 'ContainerAlreadyExistsError' diff --git a/test/storage/test_cloudfiles.py b/test/storage/test_cloudfiles.py index f7d0ea80e5..dbf7f74473 100644 --- a/test/storage/test_cloudfiles.py +++ b/test/storage/test_cloudfiles.py @@ -35,6 +35,8 @@ from test import StorageMockHttp, MockRawResponse # pylint: disable-msg=E0611 from test.file_fixtures import StorageFileFixtures # pylint: disable-msg=E0611 +current_hash = None + class CloudFilesTests(unittest.TestCase): def setUp(self): @@ -279,7 +281,7 @@ def upload_file(self, response, file_path, chunked=False, try: self.driver.upload_object(file_path=file_path, container=container, object_name=object_name, - file_hash='footest123') + verify_hash=True) except ObjectHashMismatchError: pass else: @@ -598,16 +600,19 @@ def _v1_MossoCloudFS_foo_bar_container_foo_test_upload( # test_object_upload_success body = '' - headers = copy.deepcopy(self.base_headers) - headers.update(headers) + headers = {} + headers.update(self.base_headers) + headers['etag'] = 'hash343hhash89h932439jsaa89' return (httplib.CREATED, body, headers, httplib.responses[httplib.OK]) def _v1_MossoCloudFS_foo_bar_container_foo_test_upload_INVALID_HASH( self, method, url, body, headers): # test_object_upload_invalid_hash body = '' - headers = self.base_headers - return (httplib.UNPROCESSABLE_ENTITY, body, headers, + headers = {} + headers.update(self.base_headers) + headers['etag'] = 'foobar' + return (httplib.CREATED, body, headers, httplib.responses[httplib.OK]) def _v1_MossoCloudFS_foo_bar_container_foo_bar_object( @@ -641,10 +646,13 @@ def _v1_MossoCloudFS_foo_bar_container_foo_test_stream_data( self, method, url, body, headers): # test_upload_object_via_stream_success + headers = {} + headers.update(self.base_headers) + headers['etag'] = '577ef1154f3240ad5b9b413aa7346a1e' body = 'test' return (httplib.CREATED, body, - self.base_headers, + headers, httplib.responses[httplib.OK]) if __name__ == '__main__': diff --git a/test/storage/test_s3.py b/test/storage/test_s3.py index 5395daf4b5..e529f5bc02 100644 --- a/test/storage/test_s3.py +++ b/test/storage/test_s3.py @@ -277,7 +277,7 @@ def test_upload_object_invalid_ex_storage_class(self): try: self.driver.upload_object(file_path=file_path, container=container, object_name=object_name, - file_hash='0cc175b9c0f1b6a831c399e269772661', + verify_hash=True, ex_storage_class='invalid-class') except ValueError, e: self.assertTrue(str(e).lower().find('invalid storage class') != -1) @@ -302,7 +302,7 @@ def upload_file(self, response, file_path, chunked=False, try: self.driver.upload_object(file_path=file_path, container=container, object_name=object_name, - file_hash='0cc175b9c0f1b6a831c399e269772661') + verify_hash=True) except ObjectHashMismatchError: pass else: @@ -328,7 +328,7 @@ def upload_file(self, response, file_path, chunked=False, try: self.driver.upload_object(file_path=file_path, container=container, object_name=object_name, - file_hash='0cc175b9c0f1b6a831c399e269772661') + verify_hash=True) except ObjectHashMismatchError: pass else: @@ -351,7 +351,7 @@ def upload_file(self, response, file_path, chunked=False, obj = self.driver.upload_object(file_path=file_path, container=container, object_name=object_name, extra=extra, - file_hash='0cc175b9c0f1b6a831c399e269772661') + verify_hash=True) self.assertEqual(obj.name, 'foo_test_upload') self.assertEqual(obj.size, 1000) self.assertTrue('some-value' in obj.meta_data) @@ -564,8 +564,10 @@ def _foo_bar_container_foo_bar_object(self, method, url, body, headers): def _foo_bar_container_foo_test_upload_INVALID_HASH1(self, method, url, body, headers): body = '' + headers = {} + headers['etag'] = '"foobar"' # test_upload_object_invalid_hash1 - return (httplib.BAD_REQUEST, + return (httplib.OK, body, headers, httplib.responses[httplib.OK]) From db2ab549ba1d3523d9eaeae536fb191c857aa88b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sun, 15 May 2011 12:56:48 +0000 Subject: [PATCH 43/90] Make get_meta_data and extension method and update affected tests. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103357 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/storage/base.py | 10 -------- libcloud/storage/drivers/cloudfiles.py | 34 +++++++++++++------------- test/storage/test_cloudfiles.py | 9 ++++--- test/storage/test_s3.py | 8 ------ 4 files changed, 23 insertions(+), 38 deletions(-) diff --git a/libcloud/storage/base.py b/libcloud/storage/base.py index a66db7f562..b7dd09e99c 100644 --- a/libcloud/storage/base.py +++ b/libcloud/storage/base.py @@ -178,16 +178,6 @@ def __init__(self, key, secret=None, secure=True, host=None, port=None): self.connection.driver = self self.connection.connect() - def get_meta_data(self): - """ - Return account meta data - total number of containers, objects and - number of bytes currently used. - - @return A C{dict} with account meta data. - """ - raise NotImplementedError( - 'get_account_meta_data not implemented for this driver') - def list_containters(self): raise NotImplementedError( 'list_containers not implemented for this driver') diff --git a/libcloud/storage/drivers/cloudfiles.py b/libcloud/storage/drivers/cloudfiles.py index c3e70f69a5..20a53012f6 100644 --- a/libcloud/storage/drivers/cloudfiles.py +++ b/libcloud/storage/drivers/cloudfiles.py @@ -147,23 +147,6 @@ class CloudFilesStorageDriver(StorageDriver): connectionCls = CloudFilesConnection hash_type = 'md5' - def get_meta_data(self): - response = self.connection.request('', method='HEAD') - - if response.status == httplib.NO_CONTENT: - container_count = response.headers.get( - 'x-account-container-count', 'unknown') - object_count = response.headers.get( - 'x-account-object-count', 'unknown') - bytes_used = response.headers.get( - 'x-account-bytes-used', 'unknown') - - return { 'container_count': int(container_count), - 'object_count': int(object_count), - 'bytes_used': int(bytes_used) } - - raise LibcloudError('Unexpected status code: %s' % (response.status)) - def list_containers(self): response = self.connection.request('') @@ -351,6 +334,23 @@ def delete_object(self, obj): raise LibcloudError('Unexpected status code: %s' % (response.status)) + def ex_get_meta_data(self): + response = self.connection.request('', method='HEAD') + + if response.status == httplib.NO_CONTENT: + container_count = response.headers.get( + 'x-account-container-count', 'unknown') + object_count = response.headers.get( + 'x-account-object-count', 'unknown') + bytes_used = response.headers.get( + 'x-account-bytes-used', 'unknown') + + return { 'container_count': int(container_count), + 'object_count': int(object_count), + 'bytes_used': int(bytes_used) } + + raise LibcloudError('Unexpected status code: %s' % (response.status)) + def _put_object(self, container, object_name, upload_func, upload_func_kwargs, extra=None, file_path=None, iterator=None, verify_hash=True): diff --git a/test/storage/test_cloudfiles.py b/test/storage/test_cloudfiles.py index dbf7f74473..13a2700ee2 100644 --- a/test/storage/test_cloudfiles.py +++ b/test/storage/test_cloudfiles.py @@ -52,9 +52,6 @@ def setUp(self): def tearDown(self): self._remove_test_file() - def test_get_meta_data(self): - self.driver.get_meta_data() - def test_invalid_json_throws_exception(self): CloudFilesMockHttp.type = 'MALFORMED_JSON' try: @@ -399,6 +396,12 @@ def test_delete_object_not_found(self): else: self.fail('Object does not exist but an exception was not thrown') + def test_ex_get_meta_data(self): + meta_data = self.driver.ex_get_meta_data() + self.assertTrue(isinstance(meta_data, dict)) + self.assertTrue('object_count' in meta_data) + self.assertTrue('container_count' in meta_data) + self.assertTrue('bytes_used' in meta_data) def _remove_test_file(self): file_path = os.path.abspath(__file__) + '.temp' diff --git a/test/storage/test_s3.py b/test/storage/test_s3.py index e529f5bc02..8e31c00e24 100644 --- a/test/storage/test_s3.py +++ b/test/storage/test_s3.py @@ -73,14 +73,6 @@ def test_bucket_is_located_in_different_region(self): else: self.fail('Exception was not thrown') - def test_get_meta_data(self): - try: - self.driver.get_meta_data() - except NotImplementedError: - pass - else: - self.fail('Exception was not thrown') - def test_list_containers_empty(self): S3MockHttp.type = 'list_containers_EMPTY' containers = self.driver.list_containers() From 63f64542b79fe31855f9b17baf45feadb1fd1cf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sun, 15 May 2011 13:14:47 +0000 Subject: [PATCH 44/90] Fix a bug in the gogrid driver and add a test case for it. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103368 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/drivers/rackspace.py | 2 ++ test/loadbalancer/test_gogrid.py | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/libcloud/loadbalancer/drivers/rackspace.py b/libcloud/loadbalancer/drivers/rackspace.py index 40bc204994..babbae8445 100644 --- a/libcloud/loadbalancer/drivers/rackspace.py +++ b/libcloud/loadbalancer/drivers/rackspace.py @@ -90,6 +90,8 @@ def list_balancers(self): def create_balancer(self, name, members, protocol='http', port=80, algorithm=DEFAULT_ALGORITHM): + algorithm = self._algorithm_to_value(algorithm) + balancer_object = {"loadBalancer": {"name": name, "port": port, diff --git a/test/loadbalancer/test_gogrid.py b/test/loadbalancer/test_gogrid.py index 9d5b422b25..980d9e0a0a 100644 --- a/test/loadbalancer/test_gogrid.py +++ b/test/loadbalancer/test_gogrid.py @@ -2,6 +2,7 @@ import os.path import sys import unittest +from urlparse import urlparse, parse_qsl from libcloud.loadbalancer.base import LoadBalancer, Member, Algorithm from libcloud.loadbalancer.drivers.gogrid import GoGridLBDriver @@ -83,7 +84,7 @@ def test_balancer_detach_member(self): self.assertTrue(ret) -class GoGridLBMockHttp(MockHttp): +class GoGridLBMockHttp(MockHttp, unittest.TestCase): fixtures = LoadBalancerFileFixtures('gogrid') def _api_grid_loadbalancer_list(self, method, url, body, headers): @@ -95,6 +96,9 @@ def _api_grid_ip_list(self, method, url, body, headers): return (httplib.OK, body, {}, httplib.responses[httplib.OK]) def _api_grid_loadbalancer_add(self, method, url, body, headers): + qs = dict(parse_qsl(urlparse(url).query)) + self.assertEqual(qs['loadbalancer.type'], 'round balancer') + body = self.fixtures.load('loadbalancer_add.json') return (httplib.OK, body, {}, httplib.responses[httplib.OK]) From a5c9bd226d88d91363c09feb80fea98f13ed47e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sun, 15 May 2011 13:20:08 +0000 Subject: [PATCH 45/90] Make sure the correct parameters are sent when creating a load balancer in the Rackspace driver. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103371 13f79535-47bb-0310-9956-ffa450edef68 --- test/loadbalancer/test_rackspace.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/test/loadbalancer/test_rackspace.py b/test/loadbalancer/test_rackspace.py index 297bc7f093..ac1c044e12 100644 --- a/test/loadbalancer/test_rackspace.py +++ b/test/loadbalancer/test_rackspace.py @@ -2,6 +2,12 @@ import os.path import sys import unittest +from urlparse import parse_qsl + +try: + import json +except ImportError: + import simplejson as json from libcloud.loadbalancer.base import Member, Algorithm from libcloud.loadbalancer.drivers.rackspace import RackspaceLBDriver @@ -78,7 +84,7 @@ def test_balancer_detach_member(self): self.assertTrue(ret) -class RackspaceLBMockHttp(MockHttp): +class RackspaceLBMockHttp(MockHttp, unittest.TestCase): fixtures = LoadBalancerFileFixtures('rackspace') def _v1_0(self, method, url, body, headers): @@ -99,6 +105,10 @@ def _v1_0_slug_loadbalancers(self, method, url, body, headers): body = self.fixtures.load('v1_slug_loadbalancers.json') return (httplib.OK, body, {}, httplib.responses[httplib.OK]) elif method == "POST": + body_json = json.loads(body) + self.assertEqual(body_json['loadBalancer']['protocol'], 'HTTP') + self.assertEqual(body_json['loadBalancer']['algorithm'], 'ROUND_ROBIN') + body = self.fixtures.load('v1_slug_loadbalancers_post.json') return (httplib.ACCEPTED, body, {}, httplib.responses[httplib.ACCEPTED]) From 50882e044c7a71407f92b86f96abd2dcf05c53d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sun, 15 May 2011 15:13:49 +0000 Subject: [PATCH 46/90] Add license header and update the example. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103426 13f79535-47bb-0310-9956-ffa450edef68 --- example_loadbalancer.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/example_loadbalancer.py b/example_loadbalancer.py index f7bc783386..368d5ada56 100644 --- a/example_loadbalancer.py +++ b/example_loadbalancer.py @@ -1,9 +1,24 @@ #!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import time -from libcloud.loadbalancer.base import LoadBalancer, Member +from libcloud.loadbalancer.base import LoadBalancer, Member, Algorithm from libcloud.loadbalancer.types import Provider, State from libcloud.loadbalancer.providers import get_driver @@ -19,9 +34,11 @@ def main(): # itself listens on port 80/tcp new_balancer_name = 'testlb' + os.urandom(4).encode('hex') new_balancer = driver.create_balancer(name=new_balancer_name, + algorithm=Algorithm.ROUND_ROBIN, port=80, - nodes=(Member(None, '192.168.86.1', 80), - Member(None, '192.168.86.2', 8080)) + protocol='http', + members=(Member(None, '192.168.86.1', 80), + Member(None, '192.168.86.2', 8080)) ) print new_balancer @@ -30,7 +47,7 @@ def main(): # NOTE: in real life code add timeout to not end up in # endless loop when things go wrong on provider side while True: - balancer = driver.balancer_detail(balancer=new_balancer) + balancer = driver.get_balancer(balancer_id=new_balancer.id) if balancer.state == State.RUNNING: break @@ -45,7 +62,7 @@ def main(): balancer.detach_node(nodes[0]) # and add another one: 10.0.0.10:1000 - print balancer.attach_node(Member(None, ip='10.0.0.10', port='1000')) + print balancer.attach_member(Member(None, ip='10.0.0.10', port='1000')) # remove the balancer driver.destroy_balancer(new_balancer) From 5233e534438a93a039f00e59a317bb06bf28412b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sun, 15 May 2011 16:14:22 +0000 Subject: [PATCH 47/90] Remove print statement. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103442 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/storage/drivers/cloudfiles.py | 1 - 1 file changed, 1 deletion(-) diff --git a/libcloud/storage/drivers/cloudfiles.py b/libcloud/storage/drivers/cloudfiles.py index 20a53012f6..87250d8962 100644 --- a/libcloud/storage/drivers/cloudfiles.py +++ b/libcloud/storage/drivers/cloudfiles.py @@ -378,7 +378,6 @@ def _put_object(self, container, object_name, upload_func, response = result_dict['response'].response bytes_transferred = result_dict['bytes_transferred'] - print result_dict['data_hash'] server_hash = result_dict['response'].headers.get('etag', None) if response.status == httplib.EXPECTATION_FAILED: From a8d0550457d7acd1ff986ed9a4d844f4fc494fba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sun, 15 May 2011 16:20:28 +0000 Subject: [PATCH 48/90] Properly set object hash value when using get_object method in the CloudFiles storage driver. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103445 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/storage/drivers/cloudfiles.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/libcloud/storage/drivers/cloudfiles.py b/libcloud/storage/drivers/cloudfiles.py index 87250d8962..4d67b54374 100644 --- a/libcloud/storage/drivers/cloudfiles.py +++ b/libcloud/storage/drivers/cloudfiles.py @@ -476,10 +476,9 @@ def _headers_to_object(self, name, container, headers): key = key.replace('x-object-meta-', '') meta_data[key] = value - extra = { 'content_type': content_type, 'last_modified': last_modified, - 'etag': etag } + extra = { 'content_type': content_type, 'last_modified': last_modified } - obj = Object(name=name, size=size, hash=None, extra=extra, + obj = Object(name=name, size=size, hash=etag, extra=extra, meta_data=meta_data, container=container, driver=self) return obj From 7d2d79749cac7727dcdff16871c5e29e15a9f237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sun, 15 May 2011 16:21:07 +0000 Subject: [PATCH 49/90] Update affected test. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103446 13f79535-47bb-0310-9956-ffa450edef68 --- test/storage/test_cloudfiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/storage/test_cloudfiles.py b/test/storage/test_cloudfiles.py index 13a2700ee2..a79a86b2cb 100644 --- a/test/storage/test_cloudfiles.py +++ b/test/storage/test_cloudfiles.py @@ -109,8 +109,8 @@ def test_get_object_success(self): object_name='test_object') self.assertEqual(obj.container.name, 'test_container') self.assertEqual(obj.size, 555) + self.assertEqual(obj.hash, '6b21c4a111ac178feacf9ec9d0c71f17') self.assertEqual(obj.extra['content_type'], 'application/zip') - self.assertEqual(obj.extra['etag'], '6b21c4a111ac178feacf9ec9d0c71f17') self.assertEqual( obj.extra['last_modified'], 'Tue, 25 Jan 2011 22:01:49 GMT') self.assertEqual(obj.meta_data['foo-bar'], 'test 1') From 7198f67790ce1be1c069168ce333a353b74e6bbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sun, 15 May 2011 21:02:18 +0000 Subject: [PATCH 50/90] Properly capture errors if they happen in the RawResponse part. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103542 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/common/base.py | 70 ++++++++++++++------------ libcloud/storage/drivers/cloudfiles.py | 5 +- libcloud/storage/drivers/s3.py | 6 ++- 3 files changed, 46 insertions(+), 35 deletions(-) diff --git a/libcloud/common/base.py b/libcloud/common/base.py index 0932de9aa2..abad9d4747 100644 --- a/libcloud/common/base.py +++ b/libcloud/common/base.py @@ -25,39 +25,6 @@ from libcloud.httplib_ssl import LibcloudHTTPSConnection from httplib import HTTPConnection as LibcloudHTTPConnection -class RawResponse(object): - - def __init__(self, response=None): - self._status = None - self._response = None - self._headers = {} - self._error = None - self._reason = None - - @property - def response(self): - if not self._response: - self._response = self.connection.connection.getresponse() - return self._response - - @property - def status(self): - if not self._status: - self._status = self.response.status - return self._status - - @property - def headers(self): - if not self._headers: - self._headers = dict(self.response.getheaders()) - return self._headers - - @property - def reason(self): - if not self._reason: - self._reason = self.response.reason - return self._reason - class Response(object): """ A Base Response class to derive from. @@ -113,6 +80,43 @@ def success(self): """ return self.status == httplib.OK or self.status == httplib.CREATED +class RawResponse(Response): + + def __init__(self, response=None): + self._status = None + self._response = None + self._headers = {} + self._error = None + self._reason = None + + @property + def response(self): + if not self._response: + response = self.connection.connection.getresponse() + self._response, self.body = response, response + if not self.success(): + self.parse_error() + return self._response + + @property + def status(self): + if not self._status: + self._status = self.response.status + return self._status + + @property + def headers(self): + if not self._headers: + self._headers = dict(self.response.getheaders()) + return self._headers + + @property + def reason(self): + if not self._reason: + self._reason = self.response.reason + return self._reason + + #TODO: Move this to a better location/package class LoggingConnection(): """ diff --git a/libcloud/storage/drivers/cloudfiles.py b/libcloud/storage/drivers/cloudfiles.py index 4d67b54374..0f882f1e35 100644 --- a/libcloud/storage/drivers/cloudfiles.py +++ b/libcloud/storage/drivers/cloudfiles.py @@ -23,7 +23,7 @@ from libcloud.utils import read_in_chunks from libcloud.common.types import MalformedResponseError, LibcloudError -from libcloud.common.base import Response +from libcloud.common.base import Response, RawResponse from libcloud.storage.providers import Provider from libcloud.storage.base import Object, Container, StorageDriver @@ -78,6 +78,8 @@ def parse_body(self): return data +class CloudFilesRawResponse(CloudFilesResponse, RawResponse): + pass class CloudFilesConnection(RackspaceBaseConnection): """ @@ -85,6 +87,7 @@ class CloudFilesConnection(RackspaceBaseConnection): """ responseCls = CloudFilesResponse + rawResponseCls = CloudFilesRawResponse auth_host = None _url_key = "storage_url" diff --git a/libcloud/storage/drivers/s3.py b/libcloud/storage/drivers/s3.py index 75bef17242..805666a1f8 100644 --- a/libcloud/storage/drivers/s3.py +++ b/libcloud/storage/drivers/s3.py @@ -26,7 +26,7 @@ from libcloud.utils import fixxpath, findtext, in_development_warning from libcloud.utils import read_in_chunks from libcloud.common.types import InvalidCredsError, LibcloudError -from libcloud.common.base import ConnectionUserAndKey +from libcloud.common.base import ConnectionUserAndKey, RawResponse from libcloud.common.aws import AWSBaseResponse from libcloud.storage.base import Object, Container, StorageDriver @@ -70,6 +70,9 @@ def parse_error(self): raise LibcloudError('Unknown error. Status code: %d' % (self.status), driver=S3StorageDriver) +class S3RawResponse(S3Response, RawResponse): + pass + class S3Connection(ConnectionUserAndKey): """ Repersents a single connection to the EC2 Endpoint @@ -77,6 +80,7 @@ class S3Connection(ConnectionUserAndKey): host = 's3.amazonaws.com' responseCls = S3Response + rawResponseCls = S3RawResponse def add_default_params(self, params): expires = str(int(time.time()) + EXPIRATION_SECONDS) From 2f8cb97bb172fcf62be035d207cdd795e13410fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Mon, 16 May 2011 05:43:25 +0000 Subject: [PATCH 51/90] Update loadbalancer example. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1103612 13f79535-47bb-0310-9956-ffa450edef68 --- example_loadbalancer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example_loadbalancer.py b/example_loadbalancer.py index 368d5ada56..9ea2616330 100644 --- a/example_loadbalancer.py +++ b/example_loadbalancer.py @@ -55,11 +55,11 @@ def main(): time.sleep(30) # fetch list of nodes - nodes = balancer.list_nodes() + nodes = balancer.list_members() print nodes # remove first node - balancer.detach_node(nodes[0]) + balancer.detach_member(nodes[0]) # and add another one: 10.0.0.10:1000 print balancer.attach_member(Member(None, ip='10.0.0.10', port='1000')) From 089ba94d77221d2271679a4b000acb09510bcbd6 Mon Sep 17 00:00:00 2001 From: Roman Bogorodskiy Date: Tue, 17 May 2011 08:14:47 +0000 Subject: [PATCH 52/90] Minor style fixes for loadbalancer example. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1104046 13f79535-47bb-0310-9956-ffa450edef68 --- example_loadbalancer.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/example_loadbalancer.py b/example_loadbalancer.py index 9ea2616330..0087058d89 100644 --- a/example_loadbalancer.py +++ b/example_loadbalancer.py @@ -18,7 +18,7 @@ import os import time -from libcloud.loadbalancer.base import LoadBalancer, Member, Algorithm +from libcloud.loadbalancer.base import Member, Algorithm from libcloud.loadbalancer.types import Provider, State from libcloud.loadbalancer.providers import get_driver @@ -29,6 +29,8 @@ def main(): balancers = driver.list_balancers() + print balancers + # creating a balancer which balances traffic across two # nodes: 192.168.86.1:80 and 192.168.86.2:8080. Balancer # itself listens on port 80/tcp @@ -52,14 +54,15 @@ def main(): if balancer.state == State.RUNNING: break + print "sleeping for 30 seconds for balancers to become ready" time.sleep(30) - # fetch list of nodes - nodes = balancer.list_members() - print nodes + # fetch list of members + members = balancer.list_members() + print members - # remove first node - balancer.detach_member(nodes[0]) + # remove first member + balancer.detach_member(members[0]) # and add another one: 10.0.0.10:1000 print balancer.attach_member(Member(None, ip='10.0.0.10', port='1000')) From e30defe68d40001230a6828b9971a6818c8dcf0d Mon Sep 17 00:00:00 2001 From: Roman Bogorodskiy Date: Tue, 17 May 2011 08:15:35 +0000 Subject: [PATCH 53/90] Remove no longer used resource directory. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1104047 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/resource/__init__.py | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 libcloud/resource/__init__.py diff --git a/libcloud/resource/__init__.py b/libcloud/resource/__init__.py deleted file mode 100644 index 63ed5734fe..0000000000 --- a/libcloud/resource/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -__all__ = [ - 'lb' -] From 3373c677f03e70e9e3704ce9d1cef2516e687ea2 Mon Sep 17 00:00:00 2001 From: Roman Bogorodskiy Date: Tue, 17 May 2011 08:22:05 +0000 Subject: [PATCH 54/90] Chase rename of LB and LBNode classes for loadbalancers. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1104048 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/base.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/libcloud/loadbalancer/base.py b/libcloud/loadbalancer/base.py index c75e065c92..45b6f1691d 100644 --- a/libcloud/loadbalancer/base.py +++ b/libcloud/loadbalancer/base.py @@ -31,7 +31,7 @@ def __init__(self, id, ip, port): self.port = port def __repr__(self): - return ('' % (self.id, + return ('' % (self.id, self.ip, self.port)) class Algorithm(object): @@ -67,7 +67,7 @@ def list_members(self): return self.driver.balancer_list_members(self) def __repr__(self): - return ('' % (self.id, + return ('' % (self.id, self.name, self.state)) @@ -109,7 +109,7 @@ def list_balancers(self): """ List all loadbalancers - @return: C{list} of L{LB} objects + @return: C{list} of L{LoadBalancer} objects """ @@ -122,8 +122,8 @@ def create_balancer(self, name, port, protocol, algorithm, members): @keyword name: Name of the new load balancer (required) @type name: C{str} - @keyword members: C{list} ofL{LBNode}s to attach to balancer - @type: C{list} of L{LBNode}s + @keyword members: C{list} ofL{Member}s to attach to balancer + @type: C{list} of L{Member}s @keyword protocol: Loadbalancer protocol, defaults to http. @type: C{str} @keyword port: Port the load balancer should listen on, defaults to 80 @@ -149,12 +149,12 @@ def destroy_balancer(self, balancer): def get_balancer(self, balancer_id): """ - Return a C{LB} object. + Return a C{LoadBalancer} object. @keyword balancer_id: id of a load balancer you want to fetch @type balancer_id: C{str} - @return: C{LB} + @return: C{LoadBalancer} """ raise NotImplementedError, \ @@ -166,7 +166,7 @@ def balancer_attach_compute_node(self, balancer, node): @keyword node: Member to join to the balancer @type member: C{libcloud.compute.base.Node} - @return {LBMember} Member after joining the balancer. + @return {Member} Member after joining the balancer. """ return self.attach_member(Member(None, node.public_ip[0], balancer.port)) @@ -176,8 +176,8 @@ def balancer_attach_member(self, balancer, member): Attach a member to balancer @keyword member: Member to join to the balancer - @type member: C{LBMember} - @return {LBMember} Member after joining the balancer. + @type member: C{Member} + @return {Member} Member after joining the balancer. """ raise NotImplementedError, \ @@ -198,7 +198,7 @@ def balancer_list_members(self, balancer): """ Return list of members attached to balancer - @return: C{list} of L{LBNode}s + @return: C{list} of L{Member}s """ From 7fba13b3fa21e960c86619f668738a9b1d4485b2 Mon Sep 17 00:00:00 2001 From: Roman Bogorodskiy Date: Tue, 17 May 2011 11:03:44 +0000 Subject: [PATCH 55/90] GoGrid Compute Driver: add 16GB node size. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1104132 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/compute/drivers/gogrid.py | 7 ++++++- libcloud/data/pricing.json | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/libcloud/compute/drivers/gogrid.py b/libcloud/compute/drivers/gogrid.py index 27c9f7c2f2..850c0d75d9 100644 --- a/libcloud/compute/drivers/gogrid.py +++ b/libcloud/compute/drivers/gogrid.py @@ -61,7 +61,12 @@ 'name': '8GB', 'ram': 8192, 'disk': 480, - 'bandwidth': None} + 'bandwidth': None}, + '16GB': {'id': '16GB', + 'name': '16GB', + 'ram': 16384, + 'disk': 960, + 'bandwidth': None}, } diff --git a/libcloud/data/pricing.json b/libcloud/data/pricing.json index 0dd1f2cf67..31554b9c1b 100644 --- a/libcloud/data/pricing.json +++ b/libcloud/data/pricing.json @@ -118,7 +118,8 @@ "1GB": 0.19, "2GB": 0.38, "4GB": 0.76, - "8GB": 1.52 + "8GB": 1.52, + "16GB": 3.04 }, "gandi": { From 6aced03a5dbd348cc3f74e220ed72e1763b52f3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 11:28:19 +0000 Subject: [PATCH 56/90] Also include actual error value when calling str on Error object. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125667 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/storage/types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libcloud/storage/types.py b/libcloud/storage/types.py index 195fb40acf..62f2ec1e44 100644 --- a/libcloud/storage/types.py +++ b/libcloud/storage/types.py @@ -55,9 +55,9 @@ def __init__(self, value, driver, container_name): super(ContainerError, self).__init__(value=value, driver=driver) def __str__(self): - return ('<%s in %s, container = %s>' % + return ('<%s in %s, container=%s, value=%s>' % (self.error_type, repr(self.driver), - self.container_name)) + self.container_name, self.value)) class ObjectError(LibcloudError): error_type = 'ContainerError' From a4478d90e886b7ed26a16c25693c7e27cd290c5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 12:39:09 +0000 Subject: [PATCH 57/90] Fix a bug with using the incorrect URL when performing a raw request. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125687 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/common/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcloud/common/base.py b/libcloud/common/base.py index abad9d4747..896c87e90b 100644 --- a/libcloud/common/base.py +++ b/libcloud/common/base.py @@ -379,7 +379,7 @@ def request(self, # @TODO: Should we just pass File object as body to request method # instead of dealing with splitting and sending the file ourselves? if raw: - self.connection.putrequest(method, action) + self.connection.putrequest(method, url) for key, value in headers.iteritems(): self.connection.putheader(key, value) From edeb8d83bf0244614f5cee9bdde818c59a187c13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 13:08:32 +0000 Subject: [PATCH 58/90] Don't lowercase the amz header values in the request signature. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125691 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/storage/drivers/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcloud/storage/drivers/s3.py b/libcloud/storage/drivers/s3.py index 805666a1f8..12cd9a817c 100644 --- a/libcloud/storage/drivers/s3.py +++ b/libcloud/storage/drivers/s3.py @@ -118,7 +118,7 @@ def _get_aws_auth_param(self, method, headers, params, expires, if key.lower() in special_header_keys: special_header_values[key.lower()] = value.lower().strip() elif key.lower().startswith('x-amz-'): - amz_header_values[key.lower()] = value.lower().strip() + amz_header_values[key.lower()] = value.strip() if not special_header_values.has_key('content-md5'): special_header_values['content-md5'] = '' From 47b1ad38fd4d2d28766ac15117a39c9435f051b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 13:13:43 +0000 Subject: [PATCH 59/90] Fix Gandi.net tests so they work with Python 2.7 git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125693 13f79535-47bb-0310-9956-ffa450edef68 --- test/compute/test_gandi.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/compute/test_gandi.py b/test/compute/test_gandi.py index 00e13225d8..25bf95594a 100644 --- a/test/compute/test_gandi.py +++ b/test/compute/test_gandi.py @@ -36,7 +36,12 @@ def request(self, host, handler, request_body, verbose=0): mock = GandiMockHttp(host, 80) mock.request('POST', "%s/%s" % (handler, method)) resp = mock.getresponse() - return self._parse_response(resp.body, None) + + if sys.version[0] == '2' and sys.version[2] == '7': + response = self.parse_response(resp) + else: + response = self.parse_response(resp.body) + return response class GandiTests(unittest.TestCase): From 7bbc12b935eb175c8c5feff129c26bafd1cca2c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 13:27:20 +0000 Subject: [PATCH 60/90] Fix a bug with unittest module in Python 2.7 in the GoGrid load-balancer tests. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125695 13f79535-47bb-0310-9956-ffa450edef68 --- test/loadbalancer/test_gogrid.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/loadbalancer/test_gogrid.py b/test/loadbalancer/test_gogrid.py index 980d9e0a0a..293c1ea751 100644 --- a/test/loadbalancer/test_gogrid.py +++ b/test/loadbalancer/test_gogrid.py @@ -87,6 +87,17 @@ def test_balancer_detach_member(self): class GoGridLBMockHttp(MockHttp, unittest.TestCase): fixtures = LoadBalancerFileFixtures('gogrid') + def __init__(self, *args, **kwargs): + unittest.TestCase.__init__(self) + + if kwargs.get('host', None) and kwargs.get('port', None): + MockHttp.__init__(self, *args, **kwargs) + + def runTest(self): + # @TODO: Add a new base MockHttpTestCase class and add this method and + # constructor there + pass + def _api_grid_loadbalancer_list(self, method, url, body, headers): body = self.fixtures.load('loadbalancer_list.json') return (httplib.OK, body, {}, httplib.responses[httplib.OK]) From fa1a26debff8c857102565b6aa5e2893d412a2d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 13:32:05 +0000 Subject: [PATCH 61/90] Add a new MockHttpTestCase class which behaves the same as the base MockHttp class, but you can also use assert methods if you class inherits from this one. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125696 13f79535-47bb-0310-9956-ffa450edef68 --- test/__init__.py | 15 ++++++++++++++- test/loadbalancer/test_gogrid.py | 15 ++------------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/test/__init__.py b/test/__init__.py index 040a21790a..f534c8021f 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -12,8 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import httplib import random +import unittest from cStringIO import StringIO from urllib2 import urlparse @@ -154,6 +156,18 @@ def _example_fail(self, method, url, body, headers): return (httplib.FORBIDDEN, 'Oh Noes!', {'X-Foo': 'fail'}, httplib.responses[httplib.FORBIDDEN]) +class MockHttpTestCase(MockHttp, unittest.TestCase): + # Same as the MockHttp class, but you can also use assertions in the + # classes which inherit from this one. + def __init__(self, *args, **kwargs): + unittest.TestCase.__init__(self) + + if kwargs.get('host', None) and kwargs.get('port', None): + MockHttp.__init__(self, *args, **kwargs) + + def runTest(self): + pass + class StorageMockHttp(MockHttp): def putrequest(self, method, action): pass @@ -167,7 +181,6 @@ def endheaders(self): def send(self, data): pass - class MockRawResponse(BaseMockHttpObject): """ Mock RawResponse object suitable for testing. diff --git a/test/loadbalancer/test_gogrid.py b/test/loadbalancer/test_gogrid.py index 293c1ea751..0cb0d3b1b8 100644 --- a/test/loadbalancer/test_gogrid.py +++ b/test/loadbalancer/test_gogrid.py @@ -7,7 +7,7 @@ from libcloud.loadbalancer.base import LoadBalancer, Member, Algorithm from libcloud.loadbalancer.drivers.gogrid import GoGridLBDriver -from test import MockHttp, MockRawResponse +from test import MockHttpTestCase from test.file_fixtures import LoadBalancerFileFixtures class GoGridTests(unittest.TestCase): @@ -84,20 +84,9 @@ def test_balancer_detach_member(self): self.assertTrue(ret) -class GoGridLBMockHttp(MockHttp, unittest.TestCase): +class GoGridLBMockHttp(MockHttpTestCase): fixtures = LoadBalancerFileFixtures('gogrid') - def __init__(self, *args, **kwargs): - unittest.TestCase.__init__(self) - - if kwargs.get('host', None) and kwargs.get('port', None): - MockHttp.__init__(self, *args, **kwargs) - - def runTest(self): - # @TODO: Add a new base MockHttpTestCase class and add this method and - # constructor there - pass - def _api_grid_loadbalancer_list(self, method, url, body, headers): body = self.fixtures.load('loadbalancer_list.json') return (httplib.OK, body, {}, httplib.responses[httplib.OK]) From 295c3b4eb9df364cb7841d51128265219956f621 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 13:32:28 +0000 Subject: [PATCH 62/90] Add missing license hader. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125697 13f79535-47bb-0310-9956-ffa450edef68 --- test/loadbalancer/test_gogrid.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/loadbalancer/test_gogrid.py b/test/loadbalancer/test_gogrid.py index 0cb0d3b1b8..9cee93e7f6 100644 --- a/test/loadbalancer/test_gogrid.py +++ b/test/loadbalancer/test_gogrid.py @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import httplib import os.path import sys From d35f5288e554dcc9853564a462e4e0969452c3e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 13:36:31 +0000 Subject: [PATCH 63/90] Add another missing license header. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125698 13f79535-47bb-0310-9956-ffa450edef68 --- test/loadbalancer/test_rackspace.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/loadbalancer/test_rackspace.py b/test/loadbalancer/test_rackspace.py index ac1c044e12..26fb93f04f 100644 --- a/test/loadbalancer/test_rackspace.py +++ b/test/loadbalancer/test_rackspace.py @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import httplib import os.path import sys From 3893043931bdc3fc296b37077c37b8bd4c635b84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 13:38:50 +0000 Subject: [PATCH 64/90] Update .ratignore (Ignore loadbalancer fixtures and empty __init__.py files. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125699 13f79535-47bb-0310-9956-ffa450edef68 --- .ratignore | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.ratignore b/.ratignore index 3f0f82e0da..f113a4d195 100644 --- a/.ratignore +++ b/.ratignore @@ -6,13 +6,13 @@ CHANGES HACKING test/storage/fixtures/ test/compute/fixtures/ +test/loadbalancer/fixtures/ coverage_html_report/ .coverage .coveragerc -data/pricing.json -test/pricing_test.json - -common/__init__.py -compute/__init__.py -storage/__init__.py +libcloud/data/pricing.json +libcloud/common/__init__.py +libcloud/compute/__init__.py +libcloud/storage/__init__.py test/storage/__init__.py +test/pricing_test.json From 8cf91d748ee5244e8ffe57b389475179f7d1d9f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 17:31:09 +0000 Subject: [PATCH 65/90] Add a notice to the balancer_detach_member method. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125748 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/drivers/rackspace.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libcloud/loadbalancer/drivers/rackspace.py b/libcloud/loadbalancer/drivers/rackspace.py index babbae8445..55e7b326ee 100644 --- a/libcloud/loadbalancer/drivers/rackspace.py +++ b/libcloud/loadbalancer/drivers/rackspace.py @@ -137,6 +137,9 @@ def balancer_attach_member(self, balancer, member): return self._to_members(resp.object)[0] def balancer_detach_member(self, balancer, member): + # Loadbalancer always needs to have at least 1 member. + # Last member cannot be detached. You can only disable it or destroy the + # balancer. uri = '/loadbalancers/%s/nodes/%s' % (balancer.id, member.id) resp = self.connection.request(uri, method='DELETE') From d80a719d28a7d750dda84a42538a763ddb2f06ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 17:34:15 +0000 Subject: [PATCH 66/90] Remove type attribute from the Loadbalancer driver classes. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125749 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/drivers/gogrid.py | 1 - libcloud/loadbalancer/drivers/rackspace.py | 1 - 2 files changed, 2 deletions(-) diff --git a/libcloud/loadbalancer/drivers/gogrid.py b/libcloud/loadbalancer/drivers/gogrid.py index dd7b6db952..9121c90613 100644 --- a/libcloud/loadbalancer/drivers/gogrid.py +++ b/libcloud/loadbalancer/drivers/gogrid.py @@ -25,7 +25,6 @@ class GoGridLBDriver(BaseGoGridDriver, Driver): connectionCls = GoGridConnection - type = Provider.RACKSPACE api_name = 'gogrid_lb' name = 'GoGrid LB' diff --git a/libcloud/loadbalancer/drivers/rackspace.py b/libcloud/loadbalancer/drivers/rackspace.py index 55e7b326ee..e82e91ed79 100644 --- a/libcloud/loadbalancer/drivers/rackspace.py +++ b/libcloud/loadbalancer/drivers/rackspace.py @@ -67,7 +67,6 @@ def request(self, action, params=None, data='', headers=None, method='GET'): class RackspaceLBDriver(Driver): connectionCls = RackspaceConnection - type = Provider.RACKSPACE api_name = 'rackspace_lb' name = 'Rackspace LB' From 8771ed38f3d54e65642363a7ed55ea96606aa49f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 17:36:07 +0000 Subject: [PATCH 67/90] Rename RACKSPACE load-balancer provider constant to RACKSPACE_US. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125750 13f79535-47bb-0310-9956-ffa450edef68 --- example_loadbalancer.py | 5 +---- libcloud/loadbalancer/providers.py | 2 +- libcloud/loadbalancer/types.py | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/example_loadbalancer.py b/example_loadbalancer.py index 0087058d89..61ad5ac22c 100644 --- a/example_loadbalancer.py +++ b/example_loadbalancer.py @@ -23,7 +23,7 @@ from libcloud.loadbalancer.providers import get_driver def main(): - Rackspace = get_driver(Provider.RACKSPACE) + Rackspace = get_driver(Provider.RACKSPACE_US) driver = Rackspace('username', 'api key') @@ -64,9 +64,6 @@ def main(): # remove first member balancer.detach_member(members[0]) - # and add another one: 10.0.0.10:1000 - print balancer.attach_member(Member(None, ip='10.0.0.10', port='1000')) - # remove the balancer driver.destroy_balancer(new_balancer) diff --git a/libcloud/loadbalancer/providers.py b/libcloud/loadbalancer/providers.py index 63a99ef45d..fb12e82813 100644 --- a/libcloud/loadbalancer/providers.py +++ b/libcloud/loadbalancer/providers.py @@ -23,7 +23,7 @@ ] DRIVERS = { - Provider.RACKSPACE: + Provider.RACKSPACE_US: ('libcloud.loadbalancer.drivers.rackspace', 'RackspaceLBDriver'), Provider.GOGRID: ('libcloud.loadbalancer.drivers.gogrid', 'GoGridLBDriver'), diff --git a/libcloud/loadbalancer/types.py b/libcloud/loadbalancer/types.py index 4f84c39ef0..79c214436b 100644 --- a/libcloud/loadbalancer/types.py +++ b/libcloud/loadbalancer/types.py @@ -27,7 +27,7 @@ class LibcloudLBError(LibcloudError): pass class LibcloudLBImmutableError(LibcloudLBError): pass class Provider(object): - RACKSPACE = 0 + RACKSPACE_US = 0 GOGRID = 1 class State(object): From 4ca057b8453c953a5684a493c0580d91d6f0d997 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 17:58:19 +0000 Subject: [PATCH 68/90] Should be round robin. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125757 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/drivers/gogrid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcloud/loadbalancer/drivers/gogrid.py b/libcloud/loadbalancer/drivers/gogrid.py index 9121c90613..d761ec9a64 100644 --- a/libcloud/loadbalancer/drivers/gogrid.py +++ b/libcloud/loadbalancer/drivers/gogrid.py @@ -31,7 +31,7 @@ class GoGridLBDriver(BaseGoGridDriver, Driver): LB_STATE_MAP = { 'On': State.RUNNING, 'Unknown': State.UNKNOWN } _VALUE_TO_ALGORITHM_MAP = { - 'round balancer': Algorithm.ROUND_ROBIN, + 'round robin': Algorithm.ROUND_ROBIN, 'least connection': Algorithm.LEAST_CONNECTIONS } _ALGORITHM_TO_VALUE_MAP = reverse_dict(_VALUE_TO_ALGORITHM_MAP) From 545c7f37fcf3a946e17c9b312b6e181dccf6dcac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 18:12:50 +0000 Subject: [PATCH 69/90] Index needs to start with 1 otherwise Unexpected server error is thrown. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125759 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/drivers/gogrid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcloud/loadbalancer/drivers/gogrid.py b/libcloud/loadbalancer/drivers/gogrid.py index d761ec9a64..d314da6943 100644 --- a/libcloud/loadbalancer/drivers/gogrid.py +++ b/libcloud/loadbalancer/drivers/gogrid.py @@ -161,7 +161,7 @@ def _members_to_params(self, members): params = {} - i = 0 + i = 1 for member in members: params["realiplist.%s.ip" % i] = member.ip params["realiplist.%s.port" % i] = member.port From 82413153f82bfa093c19cece520c4d76c2d43d1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 18:15:37 +0000 Subject: [PATCH 70/90] Update gogrid test. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125760 13f79535-47bb-0310-9956-ffa450edef68 --- test/loadbalancer/test_gogrid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/loadbalancer/test_gogrid.py b/test/loadbalancer/test_gogrid.py index 9cee93e7f6..d43bcd0432 100644 --- a/test/loadbalancer/test_gogrid.py +++ b/test/loadbalancer/test_gogrid.py @@ -112,7 +112,7 @@ def _api_grid_ip_list(self, method, url, body, headers): def _api_grid_loadbalancer_add(self, method, url, body, headers): qs = dict(parse_qsl(urlparse(url).query)) - self.assertEqual(qs['loadbalancer.type'], 'round balancer') + self.assertEqual(qs['loadbalancer.type'], 'round robin') body = self.fixtures.load('loadbalancer_add.json') return (httplib.OK, body, {}, httplib.responses[httplib.OK]) From 2e643436fa15cdc501c2c650319d047a643c2796 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 18:26:41 +0000 Subject: [PATCH 71/90] Revert back to the old behavior and start with 0. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125764 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/drivers/gogrid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcloud/loadbalancer/drivers/gogrid.py b/libcloud/loadbalancer/drivers/gogrid.py index d314da6943..d761ec9a64 100644 --- a/libcloud/loadbalancer/drivers/gogrid.py +++ b/libcloud/loadbalancer/drivers/gogrid.py @@ -161,7 +161,7 @@ def _members_to_params(self, members): params = {} - i = 1 + i = 0 for member in members: params["realiplist.%s.ip" % i] = member.ip params["realiplist.%s.port" % i] = member.port From 6ddb8b235750254586437795c927c46f8fd3d125 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 18:47:58 +0000 Subject: [PATCH 72/90] Should be 'least connect'. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125765 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/drivers/gogrid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcloud/loadbalancer/drivers/gogrid.py b/libcloud/loadbalancer/drivers/gogrid.py index d761ec9a64..a6cc6e5dff 100644 --- a/libcloud/loadbalancer/drivers/gogrid.py +++ b/libcloud/loadbalancer/drivers/gogrid.py @@ -32,7 +32,7 @@ class GoGridLBDriver(BaseGoGridDriver, Driver): 'Unknown': State.UNKNOWN } _VALUE_TO_ALGORITHM_MAP = { 'round robin': Algorithm.ROUND_ROBIN, - 'least connection': Algorithm.LEAST_CONNECTIONS + 'least connect': Algorithm.LEAST_CONNECTIONS } _ALGORITHM_TO_VALUE_MAP = reverse_dict(_VALUE_TO_ALGORITHM_MAP) From 99929b6f25440fd3a914e7e6b6156e39cf106e09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 19:13:43 +0000 Subject: [PATCH 73/90] Capture unexpected server error when creating a new load-balancer and throw a more user-friendly error message. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125772 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/loadbalancer/drivers/gogrid.py | 33 ++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/libcloud/loadbalancer/drivers/gogrid.py b/libcloud/loadbalancer/drivers/gogrid.py index a6cc6e5dff..6ecc4c0e4a 100644 --- a/libcloud/loadbalancer/drivers/gogrid.py +++ b/libcloud/loadbalancer/drivers/gogrid.py @@ -14,17 +14,42 @@ # limitations under the License. import time +import httplib + +try: + import json +except ImportError: + import simplejson as json from libcloud.common.types import LibcloudError from libcloud.utils import reverse_dict -from libcloud.common.gogrid import GoGridConnection, BaseGoGridDriver +from libcloud.common.gogrid import GoGridConnection, GoGridResponse, BaseGoGridDriver from libcloud.loadbalancer.base import LoadBalancer, Member, Driver, Algorithm from libcloud.loadbalancer.base import DEFAULT_ALGORITHM -from libcloud.loadbalancer.types import Provider, State, LibcloudLBImmutableError - +from libcloud.loadbalancer.types import State, LibcloudLBImmutableError + +class GoGridLBResponse(GoGridResponse): + def success(self): + if self.status == httplib.INTERNAL_SERVER_ERROR: + # Hack, but at least this error message is more useful than + # "unexpected server error" + body = json.loads(self.body) + if body['method'] == '/grid/loadbalancer/add' and \ + len(body['list']) >= 1 and \ + body['list'][0]['message'].find('unexpected server error') != -1: + raise LibcloudError(value='You mostly likely tried to add a ' + + 'member with an IP address not assigned ' + + 'to your account', driver=self) + return super(GoGridLBResponse, self).success() + +class GoGridLBConnection(GoGridConnection): + """ + Connection class for the GoGrid load-balancer driver. + """ + responseCls = GoGridLBResponse class GoGridLBDriver(BaseGoGridDriver, Driver): - connectionCls = GoGridConnection + connectionCls = GoGridLBConnection api_name = 'gogrid_lb' name = 'GoGrid LB' From c5896f3f935eed267f8f4c8d20eb65f47f5b7d47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 19:26:03 +0000 Subject: [PATCH 74/90] Add a new test for the GoGrid LB driver. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125774 13f79535-47bb-0310-9956-ffa450edef68 --- test/loadbalancer/test_gogrid.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/test/loadbalancer/test_gogrid.py b/test/loadbalancer/test_gogrid.py index d43bcd0432..1052521cdb 100644 --- a/test/loadbalancer/test_gogrid.py +++ b/test/loadbalancer/test_gogrid.py @@ -19,6 +19,7 @@ import unittest from urlparse import urlparse, parse_qsl +from libcloud.common.types import LibcloudError from libcloud.loadbalancer.base import LoadBalancer, Member, Algorithm from libcloud.loadbalancer.drivers.gogrid import GoGridLBDriver @@ -60,6 +61,24 @@ def test_create_balancer(self): self.assertEquals(balancer.name, 'test2') self.assertEquals(balancer.id, '123') + def test_create_balancer_UNEXPECTED_ERROR(self): + # Try to create new balancer and attach members with an IP address which + # does not belong to this account + GoGridLBMockHttp.type = 'UNEXPECTED_ERROR' + + try: + self.driver.create_balancer(name='test2', + port=80, + protocol='http', + algorithm=Algorithm.ROUND_ROBIN, + members=(Member(None, '10.1.0.10', 80), + Member(None, '10.1.0.11', 80)) + ) + except LibcloudError, e: + self.assertTrue(str(e).find('tried to add a member with an IP address not assigned to your account') != -1) + else: + self.fail('Exception was not thrown') + def test_destroy_balancer(self): balancer = self.driver.list_balancers()[0] @@ -117,6 +136,13 @@ def _api_grid_loadbalancer_add(self, method, url, body, headers): body = self.fixtures.load('loadbalancer_add.json') return (httplib.OK, body, {}, httplib.responses[httplib.OK]) + def _api_grid_ip_list_UNEXPECTED_ERROR(self, method, url, body, headers): + return self._api_grid_ip_list(method, url, body, headers) + + def _api_grid_loadbalancer_add_UNEXPECTED_ERROR(self, method, url, body, headers): + body = self.fixtures.load('unexpected_error.json') + return (httplib.INTERNAL_SERVER_ERROR, body, {}, httplib.responses[httplib.OK]) + def _api_grid_loadbalancer_delete(self, method, url, body, headers): body = self.fixtures.load('loadbalancer_add.json') return (httplib.OK, body, {}, httplib.responses[httplib.OK]) From 000b23532f015e0c03477f84eb72677a3579ae94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 19:27:14 +0000 Subject: [PATCH 75/90] Forgot to commit this. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125775 13f79535-47bb-0310-9956-ffa450edef68 --- test/loadbalancer/fixtures/gogrid/unexpected_error.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 test/loadbalancer/fixtures/gogrid/unexpected_error.json diff --git a/test/loadbalancer/fixtures/gogrid/unexpected_error.json b/test/loadbalancer/fixtures/gogrid/unexpected_error.json new file mode 100644 index 0000000000..87ed4e56ba --- /dev/null +++ b/test/loadbalancer/fixtures/gogrid/unexpected_error.json @@ -0,0 +1 @@ +{"summary":{"total":1,"start":0,"returned":1},"status":"failure","method":"/grid/loadbalancer/add","list":[{"message":"An unexpected server error has occured. Please email this error to apisupport@gogrid.com. Error Message : null","object":"error","errorcode":"UnexpectedException"}]} From 3d3a73ed99a7c19bf353ac228aec12f9eabe7e6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 19:37:17 +0000 Subject: [PATCH 76/90] pyflakes cleanup. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125776 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/compute/drivers/opsource.py | 11 ++++------- test/compute/test_elastichosts.py | 1 - test/compute/test_gogrid.py | 4 ++-- test/loadbalancer/test_gogrid.py | 1 - test/loadbalancer/test_rackspace.py | 4 +--- 5 files changed, 7 insertions(+), 14 deletions(-) diff --git a/libcloud/compute/drivers/opsource.py b/libcloud/compute/drivers/opsource.py index 6d39b3b572..d71f1b4efb 100644 --- a/libcloud/compute/drivers/opsource.py +++ b/libcloud/compute/drivers/opsource.py @@ -295,9 +295,6 @@ def create_node(self, **kwargs): # XXX: Node sizes can be adjusted after a node is created, but cannot be # set at create time because size is part of the image definition. - size = NodeSize(id=0, name='', ram=0, disk=None, bandwidth=None, - price=0, driver=self.connection.driver) - password = None if kwargs.has_key('auth'): auth = kwargs.get('auth') @@ -328,10 +325,10 @@ def create_node(self, **kwargs): ET.SubElement(server_elm, "administratorPassword").text = password ET.SubElement(server_elm, "isStarted").text = str(ex_isStarted) - data = self.connection.request_with_orgId('server', - method='POST', - data=ET.tostring(server_elm) - ).object + self.connection.request_with_orgId('server', + method='POST', + data=ET.tostring(server_elm) + ).object # XXX: return the last node in the list that has a matching name. this # is likely but not guaranteed to be the node we just created # because opsource allows multiple nodes to have the same name diff --git a/test/compute/test_elastichosts.py b/test/compute/test_elastichosts.py index 5efb9fd665..a22abe4259 100644 --- a/test/compute/test_elastichosts.py +++ b/test/compute/test_elastichosts.py @@ -25,7 +25,6 @@ from libcloud.common.types import InvalidCredsError, MalformedResponseError from test import MockHttp -from test.compute import TestCaseMixin from test.file_fixtures import ComputeFileFixtures class ElasticHostsTestCase(unittest.TestCase): diff --git a/test/compute/test_gogrid.py b/test/compute/test_gogrid.py index e807befb17..ddeffe4b4f 100644 --- a/test/compute/test_gogrid.py +++ b/test/compute/test_gogrid.py @@ -21,7 +21,7 @@ from libcloud.common.types import LibcloudError, InvalidCredsError from libcloud.common.gogrid import GoGridIpAddress from libcloud.compute.drivers.gogrid import GoGridNodeDriver -from libcloud.compute.base import Node, NodeImage, NodeSize, NodeLocation +from libcloud.compute.base import Node, NodeImage, NodeSize from test import MockHttp # pylint: disable-msg=E0611 from test.compute import TestCaseMixin # pylint: disable-msg=E0611 @@ -69,7 +69,7 @@ def test_reboot_node_not_successful(self): node = Node(90967, None, None, None, None, self.driver) try: - ret = self.driver.reboot_node(node) + self.driver.reboot_node(node) except Exception: pass else: diff --git a/test/loadbalancer/test_gogrid.py b/test/loadbalancer/test_gogrid.py index 1052521cdb..5db7ba9351 100644 --- a/test/loadbalancer/test_gogrid.py +++ b/test/loadbalancer/test_gogrid.py @@ -14,7 +14,6 @@ # limitations under the License. import httplib -import os.path import sys import unittest from urlparse import urlparse, parse_qsl diff --git a/test/loadbalancer/test_rackspace.py b/test/loadbalancer/test_rackspace.py index 26fb93f04f..0ffdf9de98 100644 --- a/test/loadbalancer/test_rackspace.py +++ b/test/loadbalancer/test_rackspace.py @@ -14,10 +14,8 @@ # limitations under the License. import httplib -import os.path import sys import unittest -from urlparse import parse_qsl try: import json @@ -27,7 +25,7 @@ from libcloud.loadbalancer.base import Member, Algorithm from libcloud.loadbalancer.drivers.rackspace import RackspaceLBDriver -from test import MockHttp, MockRawResponse +from test import MockHttp from test.file_fixtures import LoadBalancerFileFixtures class RackspaceLBTests(unittest.TestCase): From 2b89b3052c382d278148fe0ca728f93f270683ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 20:21:55 +0000 Subject: [PATCH 77/90] Update the version string. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125795 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libcloud/__init__.py b/libcloud/__init__.py index a6a5eaced7..68c57195a7 100644 --- a/libcloud/__init__.py +++ b/libcloud/__init__.py @@ -21,7 +21,7 @@ __all__ = ["__version__", "enable_debug"] -__version__ = "0.5.0-dev" +__version__ = "0.5.0" def enable_debug(fo): """ From 325b28cad057cabace117776ca546839cf2e969c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 20:31:55 +0000 Subject: [PATCH 78/90] Read the version number from the libcloud/__init__.py file git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125801 13f79535-47bb-0310-9956-ffa450edef68 --- setup.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3a21bb46bb..eae17a1884 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,13 @@ DOC_TEST_MODULES = [ 'libcloud.compute.drivers.dummy', 'libcloud.storage.drivers.dummy' ] +def read_version_string(): + version = None + sys.path.insert(0, pjoin(os.getcwd())) + from libcloud import __version__ + version = __version__ + return version + class TestCommand(Command): user_options = [] @@ -139,7 +146,7 @@ def run(self): setup( name='apache-libcloud', - version='0.4.3', + version=read_version_string(), description='A unified interface into many cloud server providers', author='Apache Software Foundation', author_email='libcloud@incubator.apache.org', From 357223aa3d30afe4a48e1b75149548d91a65564c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 20:35:33 +0000 Subject: [PATCH 79/90] Forgot to pop it. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125804 13f79535-47bb-0310-9956-ffa450edef68 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index eae17a1884..fe11f992ba 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ def read_version_string(): sys.path.insert(0, pjoin(os.getcwd())) from libcloud import __version__ version = __version__ + sys.path.pop(0) return version class TestCommand(Command): From 7f2210ff9df65c3776c97253ad50d4d78c754c9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 20:43:10 +0000 Subject: [PATCH 80/90] Add a note about api refactoring and the compute modules being moved to libcloud.compute.*. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125809 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index e3d0f82058..faa2c5eb91 100644 --- a/CHANGES +++ b/CHANGES @@ -2,6 +2,10 @@ Changes with Apache Libcloud 0.5.0 + *) Existing APIs directly on the libcloud.* module have been + deprecated and will be removed in version 0.6.0. Most methods + were moved to the libcloud.compute.* module. + *) Add new libcloud.loadbalancers API, with initial support for: - GoGrid Load Balancers - Rackspace Load Balancers @@ -24,7 +28,7 @@ Changes with Apache Libcloud 0.5.0 *) Updates to the GoGrid compute driver: - Use API version 1.0. - Remove sandbox flag. - - Add ex_list_ips() to list IP addresses assigned to the account. + - Add ex_list_ips() to list IP addresses assigned to the account. - Implement ex_edit_image method which allows changing image attributes like name, description and make image public or private. [Roman Bogorodskiy] @@ -32,7 +36,7 @@ Changes with Apache Libcloud 0.5.0 *) Updates to the Amazon EC2 compute driver: - When creating a Node, use the name argument to set a Tag with the value. [Tomaz Muraus] - - Add extension method for modifying node attributes and changing the + - Add extension method for modifying node attributes and changing the node size. [Tomaz Muraus] - Add support for the new Amazon Region (Tokyo). [Tomaz Muraus] - Added ex_create_tags and ex_delete_tags. [Brandon Rhodes] From 045bbdea965e72906de20de6da94b0c85c089804 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 20:45:33 +0000 Subject: [PATCH 81/90] Update setup.py, include storage packages. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125810 13f79535-47bb-0310-9956-ffa450edef68 --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index fe11f992ba..3fa3f92d94 100644 --- a/setup.py +++ b/setup.py @@ -157,6 +157,8 @@ def run(self): 'libcloud.common', 'libcloud.compute', 'libcloud.compute.drivers', + 'libcloud.storage', + 'libcloud.storage.drivers', 'libcloud.drivers', 'libcloud.loadbalancer', 'libcloud.loadbalancer.drivers', From 61abb0b3c8669bc00a1484f98861a6eeb2ace34f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 20:47:30 +0000 Subject: [PATCH 82/90] Add missing data file to setup.py. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125812 13f79535-47bb-0310-9956-ffa450edef68 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3fa3f92d94..938c5a63b4 100644 --- a/setup.py +++ b/setup.py @@ -168,7 +168,7 @@ def run(self): 'libcloud.drivers': 'libcloud/drivers' }, package_data={ - 'libcloud': ['data/*.json'], + 'libcloud': ['data/*.json', 'test/pricing_test.json'], }, license='Apache License (2.0)', url='http://incubator.apache.org/libcloud/', From 9f41922d74220d6300daf5bde7cb60e6934cb4ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Sat, 21 May 2011 21:17:31 +0000 Subject: [PATCH 83/90] Update setup.py and MANIFEST.in git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1125819 13f79535-47bb-0310-9956-ffa450edef68 --- MANIFEST.in | 13 ++++++++++--- setup.py | 3 +-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 40aa1d1604..38d865cac1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,13 +1,20 @@ include LICENSE include NOTICE include DISCLAIMER -include example.py +include example_*.py include CONTRIBUTORS include CHANGES include HACKING include README +include libcloud/data/pricing.json prune test/secrets.py -include test/*.py include demos/* +include test/*.py +include test/pricing_test.json include test/secrets.py-dist -include test/fixtures/*/* \ No newline at end of file +include test/compute/*.py +include test/storage/*.py +include test/loadbalancer/*.py +include test/compute/fixtures/*/* +include test/storage/fixtures/*/* +include test/loadbalancer/fixtures/*/* diff --git a/setup.py b/setup.py index 938c5a63b4..d4f1273aea 100644 --- a/setup.py +++ b/setup.py @@ -165,10 +165,9 @@ def run(self): ], package_dir={ 'libcloud': 'libcloud', - 'libcloud.drivers': 'libcloud/drivers' }, package_data={ - 'libcloud': ['data/*.json', 'test/pricing_test.json'], + 'libcloud': ['data/*.json'] }, license='Apache License (2.0)', url='http://incubator.apache.org/libcloud/', From a3d75ea6ab0c2c2ee20209f243ddfbdd25792d93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Mon, 23 May 2011 17:42:14 +0000 Subject: [PATCH 84/90] ex_create_tags should be a pass-through in Nimbus driver since Nimbus doesn't support this method. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1126614 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/compute/drivers/ec2.py | 9 ++++++++- test/compute/test_ec2.py | 7 +++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/libcloud/compute/drivers/ec2.py b/libcloud/compute/drivers/ec2.py index 7805cd0d65..715dfac1a6 100644 --- a/libcloud/compute/drivers/ec2.py +++ b/libcloud/compute/drivers/ec2.py @@ -1006,10 +1006,17 @@ class NimbusNodeDriver(EC2NodeDriver): _instance_types = NIMBUS_INSTANCE_TYPES def ex_describe_addresses(self, nodes): - """Nimbus doesn't support elastic IPs, so this is a passthrough + """ + Nimbus doesn't support elastic IPs, so this is a passthrough """ nodes_elastic_ip_mappings = {} for node in nodes: # empty list per node nodes_elastic_ip_mappings[node.id] = [] return nodes_elastic_ip_mappings + + def ex_create_tags(self, node, tags): + """ + Nimbus doesn't support creating tags, so this is a passthrough + """ + pass diff --git a/test/compute/test_ec2.py b/test/compute/test_ec2.py index 1a38ae4767..7af49c1e84 100644 --- a/test/compute/test_ec2.py +++ b/test/compute/test_ec2.py @@ -340,5 +340,12 @@ def test_list_nodes(self): self.assertEqual(len(node.public_ip), 1) self.assertEqual(public_ips[0], '1.2.3.5') + def test_ex_create_tags(self): + # Nimbus doesn't support creating tags so this one should be a + # passthrough + node = self.driver.list_nodes()[0] + EC2MockHttp.type = 'EX_CREATE_TAGS' + self.driver.ex_create_tags(node=node, tags={'foo': 'bar'}) + if __name__ == '__main__': sys.exit(unittest.main()) From 5ff56ffc1d515ae2c791c3fc75af6497d5e0a021 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Mon, 23 May 2011 19:42:52 +0000 Subject: [PATCH 85/90] Add a base LibcloudTestCase which eventually all the test cases should inherit from. This class will provide some utility assertion methods. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1126681 13f79535-47bb-0310-9956-ffa450edef68 --- test/__init__.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test/__init__.py b/test/__init__.py index f534c8021f..1ec9a52bde 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -21,6 +21,29 @@ from urllib2 import urlparse from cgi import parse_qs +class LibcloudTestCase(unittest.TestCase): + def __init__(self, *args, **kwargs): + self._visited_urls = [] + self._executed_mock_methods = [] + super(LibcloudTestCase, self).__init__(*args, **kwargs) + + def setUp(self): + self._visited_urls = [] + self._executed_mock_methods = [] + + def _add_visited_url(self, url): + self._visited_urls.append(url) + + def _add_executed_mock_method(self, method_name): + self._executed_mock_methods.append(method_name) + + def assertExecutedMethodCount(self, expected): + print self._executed_mock_methods + actual = len(self._executed_mock_methods) + self.assertEqual(actual, expected, + 'expected %d, but %d mock methods were executed' + % (expected, actual)) + class multipleresponse(object): """ A decorator that allows MockHttp objects to return multi responses @@ -114,6 +137,8 @@ class MockHttp(BaseMockHttpObject): type = None use_param = None # will use this param to namespace the request function + test = None # TestCase instance which is using this mock + def __init__(self, host, port, *args, **kwargs): self.host = host self.port = port @@ -129,6 +154,11 @@ def request(self, method, url, body=None, headers=None, raw=False): use_param=self.use_param, qs=qs, path=path) meth = getattr(self, meth_name) + + if self.test and isinstance(self.test, LibcloudTestCase): + self.test._add_visited_url(url=url) + self.test._add_executed_mock_method(method_name=meth_name) + status, body, headers, reason = meth(method, url, body, headers) self.response = self.responseCls(status, body, headers, reason) From 4c5b9c4e4d209fb9c3e9cf6e55c5d02ae417c4b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Mon, 23 May 2011 19:44:59 +0000 Subject: [PATCH 86/90] Refactor EC2 test to inherit from LibcloudTestCase and use the new assertion method. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1126684 13f79535-47bb-0310-9956-ffa450edef68 --- test/compute/test_ec2.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/compute/test_ec2.py b/test/compute/test_ec2.py index 7af49c1e84..3f6754b59b 100644 --- a/test/compute/test_ec2.py +++ b/test/compute/test_ec2.py @@ -21,15 +21,16 @@ from libcloud.compute.drivers.ec2 import EC2APNENodeDriver, IdempotentParamError from libcloud.compute.base import Node, NodeImage, NodeSize, NodeLocation -from test import MockHttp +from test import MockHttp, LibcloudTestCase from test.compute import TestCaseMixin from test.file_fixtures import ComputeFileFixtures from test.secrets import EC2_ACCESS_ID, EC2_SECRET -class EC2Tests(unittest.TestCase, TestCaseMixin): +class EC2Tests(LibcloudTestCase, TestCaseMixin): def setUp(self): + EC2MockHttp.test = self EC2NodeDriver.connectionCls.conn_classes = (None, EC2MockHttp) EC2MockHttp.use_param = 'Action' EC2MockHttp.type = None @@ -335,6 +336,7 @@ def test_list_sizes(self): def test_list_nodes(self): # overridden from EC2Tests -- Nimbus doesn't support elastic IPs. node = self.driver.list_nodes()[0] + self.assertExecutedMethodCount(0) public_ips = node.public_ip self.assertEqual(node.id, 'i-4382922a') self.assertEqual(len(node.public_ip), 1) @@ -344,8 +346,8 @@ def test_ex_create_tags(self): # Nimbus doesn't support creating tags so this one should be a # passthrough node = self.driver.list_nodes()[0] - EC2MockHttp.type = 'EX_CREATE_TAGS' self.driver.ex_create_tags(node=node, tags={'foo': 'bar'}) + self.assertExecutedMethodCount(0) if __name__ == '__main__': sys.exit(unittest.main()) From 7f603e0a2ea8e5a5c5ec432016383c122d8f88ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Mon, 23 May 2011 23:02:46 +0000 Subject: [PATCH 87/90] Properly handle response errors in the Rackspace driver and only throw InvalidCredsError on 401 status code. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1126805 13f79535-47bb-0310-9956-ffa450edef68 --- libcloud/common/rackspace.py | 36 ++++++++++++++++++++++------------ test/compute/test_rackspace.py | 20 ++++++++++++++----- 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/libcloud/common/rackspace.py b/libcloud/common/rackspace.py index 953a96147b..f176012cab 100644 --- a/libcloud/common/rackspace.py +++ b/libcloud/common/rackspace.py @@ -19,7 +19,7 @@ import httplib from urllib2 import urlparse from libcloud.common.base import ConnectionUserAndKey -from libcloud.compute.types import InvalidCredsError +from libcloud.compute.types import InvalidCredsError, MalformedResponseError AUTH_HOST_US='auth.api.rackspacecloud.com' AUTH_HOST_UK='lon.auth.api.rackspacecloud.com' @@ -95,19 +95,29 @@ def _populate_hosts_and_request_paths(self): resp = conn.getresponse() - if resp.status != httplib.NO_CONTENT: - raise InvalidCredsError() - - headers = dict(resp.getheaders()) - - try: - self.server_url = headers['x-server-management-url'] - self.storage_url = headers['x-storage-url'] - self.cdn_management_url = headers['x-cdn-management-url'] - self.lb_url = self.server_url.replace("servers", "ord.loadbalancers") - self.auth_token = headers['x-auth-token'] - except KeyError: + if resp.status == httplib.NO_CONTENT: + # HTTP NO CONTENT (204): auth successful + headers = dict(resp.getheaders()) + + try: + self.server_url = headers['x-server-management-url'] + self.storage_url = headers['x-storage-url'] + self.cdn_management_url = headers['x-cdn-management-url'] + self.lb_url = self.server_url.replace("servers", "ord.loadbalancers") + self.auth_token = headers['x-auth-token'] + except KeyError, e: + # Returned 204 but has missing information in the header, something is wrong + raise MalformedResponseError('Malformed response', + body='Missing header: %s' % (str(e)), + driver=self.driver) + elif resp.status == httplib.UNAUTHORIZED: + # HTTP UNAUTHORIZED (401): auth failed raise InvalidCredsError() + else: + # Any response code != 401 or 204, something is wrong + raise MalformedResponseError('Malformed response', + body='code: %s body:%s' % (resp.status, ''.join(resp.body.readlines())), + driver=self.driver) for key in ['server_url', 'storage_url', 'cdn_management_url', 'lb_url']: diff --git a/test/compute/test_rackspace.py b/test/compute/test_rackspace.py index b46f3ddf9a..dbe92ee577 100644 --- a/test/compute/test_rackspace.py +++ b/test/compute/test_rackspace.py @@ -16,7 +16,7 @@ import unittest import httplib -from libcloud.common.types import InvalidCredsError +from libcloud.common.types import InvalidCredsError, MalformedResponseError from libcloud.compute.drivers.rackspace import RackspaceNodeDriver as Rackspace from libcloud.compute.base import Node, NodeImage, NodeSize @@ -46,8 +46,17 @@ def test_auth_missing_key(self): RackspaceMockHttp.type = 'UNAUTHORIZED_MISSING_KEY' try: self.driver = Rackspace(RACKSPACE_USER, RACKSPACE_KEY) - except InvalidCredsError, e: - self.assertEqual(True, isinstance(e, InvalidCredsError)) + except MalformedResponseError, e: + self.assertEqual(True, isinstance(e, MalformedResponseError)) + else: + self.fail('test should have thrown') + + def test_auth_server_error(self): + RackspaceMockHttp.type = 'INTERNAL_SERVER_ERROR' + try: + self.driver = Rackspace(RACKSPACE_USER, RACKSPACE_KEY) + except MalformedResponseError, e: + self.assertEqual(True, isinstance(e, MalformedResponseError)) else: self.fail('test should have thrown') @@ -197,6 +206,9 @@ def _v1_0(self, method, url, body, headers): def _v1_0_UNAUTHORIZED(self, method, url, body, headers): return (httplib.UNAUTHORIZED, "", {}, httplib.responses[httplib.UNAUTHORIZED]) + def _v1_0_INTERNAL_SERVER_ERROR(self, method, url, body, headers): + return (httplib.INTERNAL_SERVER_ERROR, "

500: Internal Server Error

", {}, httplib.responses[httplib.INTERNAL_SERVER_ERROR]) + def _v1_0_UNAUTHORIZED_MISSING_KEY(self, method, url, body, headers): headers = {'x-server-management-url': 'https://servers.api.rackspacecloud.com/v1.0/slug', 'x-auth-token': 'FE011C19-CF86-4F87-BE5D-9229145D7A06', @@ -277,7 +289,5 @@ def _v1_0_slug_shared_ip_groups_detail(self, method, url, body, headers): def _v1_0_slug_servers_3445_ips_public_67_23_21_133(self, method, url, body, headers): return (httplib.ACCEPTED, "", {}, httplib.responses[httplib.ACCEPTED]) - - if __name__ == '__main__': sys.exit(unittest.main()) From a07e1ea742eba4481c5a449f3a1e448489398d62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Mon, 23 May 2011 23:10:13 +0000 Subject: [PATCH 88/90] Update changes. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1126806 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGES b/CHANGES index faa2c5eb91..a64d5e2a2a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,15 @@ -*- coding: utf-8 -*- +Changes with Apache Libcloud 0.5.1 + *) Rackspace driver: + - Properly handle response errors and only throw InvalidCredsError if + the returned status code is 401 + [Brad Morgan] + + *) Nimbus driver: + - Fix the create_node method and make the "ex_create_tag" method a no-op, + because Nimbus doesn't support creating tags. + Changes with Apache Libcloud 0.5.0 *) Existing APIs directly on the libcloud.* module have been From ed282ffe738e7f53c7adf3c07eb6d6aa4c8dfa28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toma=C5=BE=20Muraus?= Date: Tue, 24 May 2011 10:30:24 +0000 Subject: [PATCH 89/90] Update changes and add missing driver info. git-svn-id: https://svn.apache.org/repos/asf/incubator/libcloud/trunk@1126969 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index a64d5e2a2a..f14ba13266 100644 --- a/CHANGES +++ b/CHANGES @@ -31,6 +31,7 @@ Changes with Apache Libcloud 0.5.0 - Gandi.net [Aymeric Barantal] - Nimbus [David LaBissoniere] - OpenStack [Roman Bogorodskiy] + - Opsource.net [Joe Miller] *) Added "pricing" module and improved pricing handling. [Tomaz Muraus] From d6d53a50941a2030e6bb9f0e7f8274f4e599a074 Mon Sep 17 00:00:00 2001 From: Philip Schwartz Date: Thu, 26 May 2011 15:45:06 -0400 Subject: [PATCH 90/90] Added openstack base connection class, swift storage driver, and tests for the swift driver. --- libcloud/common/openstack.py | 122 ++++ libcloud/storage/drivers/swift.py | 450 ++++++++++++ libcloud/storage/providers.py | 2 + libcloud/storage/types.py | 2 + .../swift/list_container_objects.json | 14 + .../swift/list_container_objects_empty.json | 1 + .../fixtures/swift/list_containers.json | 5 + .../fixtures/swift/list_containers_empty.json | 1 + test/storage/fixtures/swift/meta_data.json | 1 + test/storage/test_swift.py | 658 ++++++++++++++++++ 10 files changed, 1256 insertions(+) create mode 100644 libcloud/common/openstack.py create mode 100644 libcloud/storage/drivers/swift.py create mode 100644 test/storage/fixtures/swift/list_container_objects.json create mode 100644 test/storage/fixtures/swift/list_container_objects_empty.json create mode 100644 test/storage/fixtures/swift/list_containers.json create mode 100644 test/storage/fixtures/swift/list_containers_empty.json create mode 100644 test/storage/fixtures/swift/meta_data.json create mode 100644 test/storage/test_swift.py diff --git a/libcloud/common/openstack.py b/libcloud/common/openstack.py new file mode 100644 index 0000000000..c3d5b0194a --- /dev/null +++ b/libcloud/common/openstack.py @@ -0,0 +1,122 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Common utilities for Rackspace Cloud Servers and Cloud Files +""" +import httplib +from urllib2 import urlparse +from libcloud.common.base import ConnectionUserAndKey +from libcloud.compute.types import InvalidCredsError, MalformedResponseError + +AUTH_API_VERSION = 'v1.0' + +__all__ = [ + "OpenstackBaseConnection", + ] + +class OpenstackBaseConnection(ConnectionUserAndKey): + def __init__(self, user_id, key, secure): + self.cdn_management_url = None + self.storage_url = None + self.auth_token = None + self.__host = None + super(OpenstackBaseConnection, self).__init__( + user_id, key, secure=secure) + + def add_default_headers(self, headers): + headers['X-Auth-Token'] = self.auth_token + headers['Accept'] = self.accept_format + return headers + + @property + def request_path(self): + return self._get_request_path(url_key=self._url_key) + + @property + def host(self): + # Default to server_host + return self._get_host(url_key=self._url_key) + + def _get_request_path(self, url_key): + value_key = '__request_path_%s' % (url_key) + value = getattr(self, value_key, None) + + if not value: + self._populate_hosts_and_request_paths() + value = getattr(self, value_key, None) + + return value + + def _get_host(self, url_key): + value_key = '__%s' % (url_key) + value = getattr(self, value_key, None) + + if not value: + self._populate_hosts_and_request_paths() + value = getattr(self, value_key, None) + + return value + + def _populate_hosts_and_request_paths(self): + """ + Rackspace uses a separate host for API calls which is only provided + after an initial authentication request. If we haven't made that + request yet, do it here. Otherwise, just return the management host. + """ + if not self.auth_token: + # Initial connection used for authentication + conn = self.conn_classes[self.secure]( + self.auth_host, self.port[self.secure]) + conn.request( + method='GET', + url='/%s' % (AUTH_API_VERSION), + headers={ + 'X-Auth-User': self.user_id, + 'X-Auth-Key': self.key + } + ) + + resp = conn.getresponse() + + if resp.status == httplib.NO_CONTENT: + # HTTP NO CONTENT (204): auth successful + headers = dict(resp.getheaders()) + + try: + self.storage_url = headers['x-storage-url'] + self.auth_token = headers['x-auth-token'] + except KeyError, e: + # Returned 204 but has missing information in the header, something is wrong + raise MalformedResponseError('Malformed response', + body='Missing header: %s' % (str(e)), + driver=self.driver) + elif resp.status == httplib.UNAUTHORIZED: + # HTTP UNAUTHORIZED (401): auth failed + raise InvalidCredsError() + else: + # Any response code != 401 or 204, something is wrong + raise MalformedResponseError('Malformed response', + body='code: %s body:%s' % (resp.status, ''.join(resp.body.readlines())), + driver=self.driver) + + for key in ['storage_url']: + scheme, server, request_path, param, query, fragment = ( + urlparse.urlparse(getattr(self, key))) + # Set host to where we want to make further requests to + setattr(self, '__%s' % (key), server) + setattr(self, '__request_path_%s' % (key), request_path) + + conn.close() diff --git a/libcloud/storage/drivers/swift.py b/libcloud/storage/drivers/swift.py new file mode 100644 index 0000000000..16d7fe5b10 --- /dev/null +++ b/libcloud/storage/drivers/swift.py @@ -0,0 +1,450 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import httplib +import urllib + +try: + import json +except: + import simplejson as json + +from libcloud.utils import read_in_chunks +from libcloud.common.types import MalformedResponseError, LibcloudError +from libcloud.common.base import Response, RawResponse + +from libcloud.storage.providers import Provider +from libcloud.storage.base import Object, Container, StorageDriver +from libcloud.storage.types import ContainerAlreadyExistsError +from libcloud.storage.types import ContainerDoesNotExistError +from libcloud.storage.types import ContainerIsNotEmptyError +from libcloud.storage.types import ObjectDoesNotExistError +from libcloud.storage.types import ObjectHashMismatchError +from libcloud.storage.types import InvalidContainerNameError + +from libcloud.common.openstack import OpenstackBaseConnection + +API_VERSION = 'v1.0' + +class SwiftResponse(Response): + + valid_response_codes = [ httplib.NOT_FOUND, httplib.CONFLICT ] + + def success(self): + i = int(self.status) + return i >= 200 and i <= 299 or i in self.valid_response_codes + + def parse_body(self): + if not self.body: + return None + + if 'content-type' in self.headers: + key = 'content-type' + elif 'Content-Type' in self.headers: + key = 'Content-Type' + else: + raise LibcloudError('Missing content-type header') + + content_type = self.headers[key] + if content_type.find(';') != -1: + content_type = content_type.split(';')[0] + + if content_type == 'application/json': + try: + data = json.loads(self.body) + except: + raise MalformedResponseError('Failed to parse JSON', + body=self.body, + driver=SwiftStorageDriver) + elif content_type == 'text/plain': + data = self.body + else: + data = self.body + + return data + +class SwiftRawResponse(SwiftResponse, RawResponse): + pass + +class SwiftConnection(OpenstackBaseConnection): + """ + Base connection class for the Cloudfiles driver. + """ + + responseCls = SwiftResponse + rawResponseCls = SwiftRawResponse + auth_host = None + _url_key = "storage_url" + + def __init__(self, user_id, key, secure=True): + super(SwiftConnection, self).__init__(user_id, key, secure=secure) + self.api_version = API_VERSION + self.accept_format = 'application/json' + + def request(self, action, params=None, data='', headers=None, method='GET', raw=False): + if not headers: + headers = {} + if not params: + params = {} + + # Due to first-run authentication request, we may not have a path + if self.request_path: + action = self.request_path + action + params['format'] = 'json' + if method in [ 'POST', 'PUT' ]: + headers.update({'Content-Type': 'application/json; charset=UTF-8'}) + + return super(SwiftConnection, self).request( + action=action, + params=params, data=data, + method=method, headers=headers, + raw=raw + ) + +class SwiftOSConnection(SwiftConnection): + """ + Connection class for the Cloudfiles US endpoint. + """ + def __init__(self, user_id, key, host, port, secure=True): + super(SwiftOSConnection, self).__init__(user_id,key, secure=secure) + self.auth_host = host + self.port = (port,port) + + +class SwiftStorageDriver(StorageDriver): + """ + Base Swift driver. + + You should never create an instance of this class directly but use US/US + class. + """ + name = 'Swift' + connectionCls = SwiftConnection + hash_type = 'md5' + + def list_containers(self): + response = self.connection.request('') + + if response.status == httplib.NO_CONTENT: + return [] + elif response.status == httplib.OK: + return self._to_container_list(json.loads(response.body)) + + raise LibcloudError('Unexpected status code: %s' % (response.status)) + + def list_container_objects(self, container): + response = self.connection.request('/%s' % (container.name)) + + if response.status == httplib.NO_CONTENT: + # Empty or inexistent container + return [] + elif response.status == httplib.OK: + return self._to_object_list(json.loads(response.body), container) + + raise LibcloudError('Unexpected status code: %s' % (response.status)) + + def get_container(self, container_name): + response = self.connection.request('/%s' % (container_name), + method='HEAD') + + if response.status == httplib.NO_CONTENT: + container = self._headers_to_container( + container_name, response.headers) + return container + elif response.status == httplib.NOT_FOUND: + raise ContainerDoesNotExistError(None, self, container_name) + + raise LibcloudError('Unexpected status code: %s' % (response.status)) + + def get_object(self, container_name, object_name): + container = self.get_container(container_name) + response = self.connection.request('/%s/%s' % (container_name, + object_name), + method='HEAD') + if response.status in [ httplib.OK, httplib.NO_CONTENT ]: + obj = self._headers_to_object( + object_name, container, response.headers) + return obj + elif response.status == httplib.NOT_FOUND: + raise ObjectDoesNotExistError(None, self, object_name) + + raise LibcloudError('Unexpected status code: %s' % (response.status)) + + def create_container(self, container_name): + container_name = self._clean_container_name(container_name) + response = self.connection.request( + '/%s' % (container_name), method='PUT') + + if response.status == httplib.CREATED: + # Accepted mean that container is not yet created but it will be + # eventually + extra = { 'object_count': 0 } + container = Container(name=container_name, extra=extra, driver=self) + + return container + elif response.status == httplib.ACCEPTED: + error = ContainerAlreadyExistsError(None, self, container_name) + raise error + + raise LibcloudError('Unexpected status code: %s' % (response.status)) + + def delete_container(self, container): + name = self._clean_container_name(container.name) + + # Only empty container can be deleted + response = self.connection.request('/%s' % (name), method='DELETE') + + if response.status == httplib.NO_CONTENT: + return True + elif response.status == httplib.NOT_FOUND: + raise ContainerDoesNotExistError(value='', + container_name=name, driver=self) + elif response.status == httplib.CONFLICT: + # @TODO: Add "delete_all_objects" parameter? + raise ContainerIsNotEmptyError(value='', + container_name=name, driver=self) + + def download_object(self, obj, destination_path, overwrite_existing=False, + delete_on_failure=True): + container_name = obj.container.name + object_name = obj.name + response = self.connection.request('/%s/%s' % (container_name, + object_name), + method='GET', raw=True) + + return self._get_object(obj=obj, callback=self._save_object, + response=response, + callback_kwargs={'obj': obj, + 'response': response.response, + 'destination_path': destination_path, + 'overwrite_existing': overwrite_existing, + 'delete_on_failure': delete_on_failure}, + success_status_code=httplib.OK) + + def download_object_as_stream(self, obj, chunk_size=None): + container_name = obj.container.name + object_name = obj.name + response = self.connection.request('/%s/%s' % (container_name, + object_name), + method='GET', raw=True) + + return self._get_object(obj=obj, callback=read_in_chunks, + response=response, + callback_kwargs={ 'iterator': response.response, + 'chunk_size': chunk_size}, + success_status_code=httplib.OK) + + def upload_object(self, file_path, container, object_name, extra=None, + verify_hash=True): + """ + Upload an object. + + Note: This will override file with a same name if it already exists. + """ + upload_func = self._upload_file + upload_func_kwargs = { 'file_path': file_path } + + return self._put_object(container=container, object_name=object_name, + upload_func=upload_func, + upload_func_kwargs=upload_func_kwargs, + extra=extra, file_path=file_path, + verify_hash=verify_hash) + + def upload_object_via_stream(self, iterator, + container, object_name, extra=None): + if isinstance(iterator, file): + iterator = iter(iterator) + + upload_func = self._stream_data + upload_func_kwargs = { 'iterator': iterator } + + return self._put_object(container=container, object_name=object_name, + upload_func=upload_func, + upload_func_kwargs=upload_func_kwargs, + extra=extra, iterator=iterator) + + def delete_object(self, obj): + container_name = self._clean_container_name(obj.container.name) + object_name = self._clean_object_name(obj.name) + + response = self.connection.request( + '/%s/%s' % (container_name, object_name), method='DELETE') + + if response.status == httplib.NO_CONTENT: + return True + elif response.status == httplib.NOT_FOUND: + raise ObjectDoesNotExistError(value='', object_name=object_name, + driver=self) + + raise LibcloudError('Unexpected status code: %s' % (response.status)) + + def ex_get_meta_data(self): + response = self.connection.request('', method='HEAD') + + if response.status == httplib.NO_CONTENT: + container_count = response.headers.get( + 'x-account-container-count', 'unknown') + object_count = response.headers.get( + 'x-account-object-count', 'unknown') + bytes_used = response.headers.get( + 'x-account-bytes-used', 'unknown') + + return { 'container_count': int(container_count), + 'object_count': int(object_count), + 'bytes_used': int(bytes_used) } + + raise LibcloudError('Unexpected status code: %s' % (response.status)) + + def _put_object(self, container, object_name, upload_func, + upload_func_kwargs, extra=None, file_path=None, + iterator=None, verify_hash=True): + extra = extra or {} + container_name_cleaned = self._clean_container_name(container.name) + object_name_cleaned = self._clean_object_name(object_name) + content_type = extra.get('content_type', None) + meta_data = extra.get('meta_data', None) + + headers = {} + if meta_data: + for key, value in meta_data.iteritems(): + key = 'X-Object-Meta-%s' % (key) + headers[key] = value + + request_path = '/%s/%s' % (container_name_cleaned, object_name_cleaned) + result_dict = self._upload_object(object_name=object_name, + content_type=content_type, + upload_func=upload_func, + upload_func_kwargs=upload_func_kwargs, + request_path=request_path, + request_method='PUT', + headers=headers, file_path=file_path, + iterator=iterator) + + response = result_dict['response'].response + bytes_transferred = result_dict['bytes_transferred'] + server_hash = result_dict['response'].headers.get('etag', None) + + if response.status == httplib.EXPECTATION_FAILED: + raise LibcloudError(value='Missing content-type header', + driver=self) + elif verify_hash and not server_hash: + raise LibcloudError(value='Server didn\'t return etag', + driver=self) + elif (verify_hash and result_dict['data_hash'] != server_hash): + raise ObjectHashMismatchError( + value=('MD5 hash checksum does not match (expected=%s, ' + + 'actual=%s)') % (result_dict['data_hash'], server_hash), + object_name=object_name, driver=self) + elif response.status == httplib.CREATED: + obj = Object( + name=object_name, size=bytes_transferred, hash=server_hash, + extra=None, meta_data=meta_data, container=container, + driver=self) + + return obj + else: + # @TODO: Add test case for this condition (probably 411) + raise LibcloudError('status_code=%s' % (response.status), + driver=self) + + def _clean_container_name(self, name): + """ + Clean container name. + """ + if name.startswith('/'): + name = name[1:] + name = urllib.quote(name) + + if name.find('/') != -1: + raise InvalidContainerNameError(value='Container name cannot' + ' contain slashes', + container_name=name, driver=self) + + if len(name) > 256: + raise InvalidContainerNameError(value='Container name cannot be' + ' longer than 256 bytes', + container_name=name, driver=self) + + + return name + + def _clean_object_name(self, name): + name = urllib.quote(name) + return name + + def _to_container_list(self, response): + # @TODO: Handle more then 10k containers - use "lazy list"? + containers = [] + + for container in response: + extra = { 'object_count': int(container['count']), + 'size': int(container['bytes'])} + containers.append(Container(name=container['name'], extra=extra, + driver=self)) + + return containers + + def _to_object_list(self, response, container): + objects = [] + + for obj in response: + name = obj['name'] + size = int(obj['bytes']) + hash = obj['hash'] + extra = { 'content_type': obj['content_type'], + 'last_modified': obj['last_modified'] } + objects.append(Object( + name=name, size=size, hash=hash, extra=extra, + meta_data=None, container=container, driver=self)) + + return objects + + def _headers_to_container(self, name, headers): + size = int(headers.get('x-container-bytes-used', 0)) + object_count = int(headers.get('x-container-object-count', 0)) + + extra = { 'object_count': object_count, + 'size': size } + container = Container(name=name, extra=extra, driver=self) + return container + + def _headers_to_object(self, name, container, headers): + size = int(headers.pop('content-length', 0)) + last_modified = headers.pop('last-modified', None) + etag = headers.pop('etag', None) + content_type = headers.pop('content-type', None) + + meta_data = {} + for key, value in headers.iteritems(): + if key.find('x-object-meta-') != -1: + key = key.replace('x-object-meta-', '') + meta_data[key] = value + + extra = { 'content_type': content_type, 'last_modified': last_modified } + + obj = Object(name=name, size=size, hash=etag, extra=extra, + meta_data=meta_data, container=container, driver=self) + return obj + + +class SwiftOSStorageDriver(SwiftStorageDriver): + """ + Cloudfiles storage driver for the US endpoint. + """ + + type = Provider.SWIFT_OS + name = 'Swift (US)' + connectionCls = SwiftOSConnection + diff --git a/libcloud/storage/providers.py b/libcloud/storage/providers.py index 43d9b32775..1102b93d1e 100644 --- a/libcloud/storage/providers.py +++ b/libcloud/storage/providers.py @@ -23,6 +23,8 @@ ('libcloud.storage.drivers.cloudfiles', 'CloudFilesUSStorageDriver'), Provider.CLOUDFILES_UK: ('libcloud.storage.drivers.cloudfiles', 'CloudFilesUKStorageDriver'), + Provider.SWIFT: + ('libcloud.storage.drivers.swift', 'SwiftOSStorageDriver'), Provider.S3: ('libcloud.storage.drivers.s3', 'S3StorageDriver'), Provider.S3_US_WEST: diff --git a/libcloud/storage/types.py b/libcloud/storage/types.py index 62f2ec1e44..54dd7d0e2b 100644 --- a/libcloud/storage/types.py +++ b/libcloud/storage/types.py @@ -37,6 +37,7 @@ class Provider(object): @cvar S3_EU_WEST: Amazon S3 EU West (Ireland) @cvar S3_AP_SOUTHEAST_HOST: Amazon S3 Asia South East (Singapore) @cvar S3_AP_NORTHEAST_HOST: Amazon S3 Asia South East (Tokyo) + @cvar SWIFT: Swfit provider allowing the usage of OpenStack Swift """ DUMMY = 0 CLOUDFILES_US = 1 @@ -46,6 +47,7 @@ class Provider(object): S3_EU_WEST = 5 S3_AP_SOUTHEAST = 6 S3_AP_NORTHEAST = 7 + SWIFT = 8 class ContainerError(LibcloudError): error_type = 'ContainerError' diff --git a/test/storage/fixtures/swift/list_container_objects.json b/test/storage/fixtures/swift/list_container_objects.json new file mode 100644 index 0000000000..4c47200bb4 --- /dev/null +++ b/test/storage/fixtures/swift/list_container_objects.json @@ -0,0 +1,14 @@ +[ + {"name":"foo test 1","hash":"16265549b5bda64ecdaa5156de4c97cc", + "bytes":1160520,"content_type":"application/zip", + "last_modified":"2011-01-25T22:01:50.351810"}, + {"name":"foo test 2","hash":"16265549b5bda64ecdaa5156de4c97bb", + "bytes":1160520,"content_type":"application/zip", + "last_modified":"2011-01-25T22:01:50.351810"}, + {"name":"foo tes 3","hash":"16265549b5bda64ecdaa5156de4c97ee", + "bytes":1160520,"content_type":"application/zip", + "last_modified":"2011-01-25T22:01:46.549890"}, + {"name":"foo test 3","hash":"16265549b5bda64ecdaa5156de4c97ff", + "bytes":1160520,"content_type":"application/text", + "last_modified":"2011-01-25T22:01:50.351810"} +] diff --git a/test/storage/fixtures/swift/list_container_objects_empty.json b/test/storage/fixtures/swift/list_container_objects_empty.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/test/storage/fixtures/swift/list_container_objects_empty.json @@ -0,0 +1 @@ +{} diff --git a/test/storage/fixtures/swift/list_containers.json b/test/storage/fixtures/swift/list_containers.json new file mode 100644 index 0000000000..ded31c02c3 --- /dev/null +++ b/test/storage/fixtures/swift/list_containers.json @@ -0,0 +1,5 @@ +[ + {"name":"container1","count":4,"bytes":3484450}, + {"name":"container2","count":120,"bytes":340084450}, + {"name":"container3","count":0,"bytes":0} +] diff --git a/test/storage/fixtures/swift/list_containers_empty.json b/test/storage/fixtures/swift/list_containers_empty.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/test/storage/fixtures/swift/list_containers_empty.json @@ -0,0 +1 @@ +{} diff --git a/test/storage/fixtures/swift/meta_data.json b/test/storage/fixtures/swift/meta_data.json new file mode 100644 index 0000000000..5049f58492 --- /dev/null +++ b/test/storage/fixtures/swift/meta_data.json @@ -0,0 +1 @@ +{"bytes_used": 1234567, "container_count": 10, "object_count": 400} diff --git a/test/storage/test_swift.py b/test/storage/test_swift.py new file mode 100644 index 0000000000..e08e60ef1c --- /dev/null +++ b/test/storage/test_swift.py @@ -0,0 +1,658 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import os.path # pylint: disable-msg=W0404 +import sys +import copy +import unittest +import httplib + +import libcloud.utils + +from libcloud.common.types import LibcloudError, MalformedResponseError +from libcloud.storage.base import Container, Object +from libcloud.storage.types import ContainerAlreadyExistsError +from libcloud.storage.types import ContainerDoesNotExistError +from libcloud.storage.types import ContainerIsNotEmptyError +from libcloud.storage.types import ObjectDoesNotExistError +from libcloud.storage.types import ObjectHashMismatchError +from libcloud.storage.types import InvalidContainerNameError +from libcloud.storage.drivers.swift import SwiftStorageDriver +from libcloud.storage.drivers.dummy import DummyIterator + +from test import StorageMockHttp, MockRawResponse # pylint: disable-msg=E0611 +from test.file_fixtures import StorageFileFixtures # pylint: disable-msg=E0611 + +current_hash = None + +class SwiftTests(unittest.TestCase): + + def setUp(self): + SwiftStorageDriver.connectionCls.conn_classes = ( + None, SwiftMockHttp) + SwiftStorageDriver.connectionCls.rawResponseCls = \ + SwiftMockRawResponse + SwiftMockHttp.type = None + SwiftMockRawResponse.type = None + self.driver = SwiftStorageDriver('dummy', 'dummy') + self._remove_test_file() + + def tearDown(self): + self._remove_test_file() + + def test_invalid_json_throws_exception(self): + SwiftMockHttp.type = 'MALFORMED_JSON' + try: + self.driver.list_containers() + except MalformedResponseError: + pass + else: + self.fail('Exception was not thrown') + + def test_list_containers(self): + SwiftMockHttp.type = 'EMPTY' + containers = self.driver.list_containers() + self.assertEqual(len(containers), 0) + + SwiftMockHttp.type = None + containers = self.driver.list_containers() + self.assertEqual(len(containers), 3) + + container = [c for c in containers if c.name == 'container2'][0] + self.assertEqual(container.extra['object_count'], 120) + self.assertEqual(container.extra['size'], 340084450) + + def test_list_container_objects(self): + SwiftMockHttp.type = 'EMPTY' + container = Container( + name='test_container', extra={}, driver=self.driver) + objects = self.driver.list_container_objects(container=container) + self.assertEqual(len(objects), 0) + + SwiftMockHttp.type = None + objects = self.driver.list_container_objects(container=container) + self.assertEqual(len(objects), 4) + + obj = [o for o in objects if o.name == 'foo test 1'][0] + self.assertEqual(obj.hash, '16265549b5bda64ecdaa5156de4c97cc') + self.assertEqual(obj.size, 1160520) + self.assertEqual(obj.container.name, 'test_container') + + def test_get_container(self): + container = self.driver.get_container(container_name='test_container') + self.assertEqual(container.name, 'test_container') + self.assertEqual(container.extra['object_count'], 800) + self.assertEqual(container.extra['size'], 1234568) + + def test_get_container_not_found(self): + try: + self.driver.get_container(container_name='not_found') + except ContainerDoesNotExistError: + pass + else: + self.fail('Exception was not thrown') + + def test_get_object_success(self): + obj = self.driver.get_object(container_name='test_container', + object_name='test_object') + self.assertEqual(obj.container.name, 'test_container') + self.assertEqual(obj.size, 555) + self.assertEqual(obj.hash, '6b21c4a111ac178feacf9ec9d0c71f17') + self.assertEqual(obj.extra['content_type'], 'application/zip') + self.assertEqual( + obj.extra['last_modified'], 'Tue, 25 Jan 2011 22:01:49 GMT') + self.assertEqual(obj.meta_data['foo-bar'], 'test 1') + self.assertEqual(obj.meta_data['bar-foo'], 'test 2') + + def test_get_object_not_found(self): + try: + self.driver.get_object(container_name='test_container', + object_name='not_found') + except ObjectDoesNotExistError: + pass + else: + self.fail('Exception was not thrown') + + def test_create_container_success(self): + container = self.driver.create_container( + container_name='test_create_container') + self.assertTrue(isinstance(container, Container)) + self.assertEqual(container.name, 'test_create_container') + self.assertEqual(container.extra['object_count'], 0) + + def test_create_container_already_exists(self): + SwiftMockHttp.type = 'ALREADY_EXISTS' + + try: + self.driver.create_container( + container_name='test_create_container') + except ContainerAlreadyExistsError: + pass + else: + self.fail( + 'Container already exists but an exception was not thrown') + + def test_create_container_invalid_name_too_long(self): + name = ''.join([ 'x' for x in range(0, 257)]) + try: + self.driver.create_container(container_name=name) + except InvalidContainerNameError: + pass + else: + self.fail( + 'Invalid name was provided (name is too long)' + ', but exception was not thrown') + + def test_create_container_invalid_name_slashes_in_name(self): + try: + self.driver.create_container(container_name='test/slashes/') + except InvalidContainerNameError: + pass + else: + self.fail( + 'Invalid name was provided (name contains slashes)' + ', but exception was not thrown') + + def test_delete_container_success(self): + container = Container(name='foo_bar_container', extra={}, driver=self) + result = self.driver.delete_container(container=container) + self.assertTrue(result) + + def test_delete_container_not_found(self): + SwiftMockHttp.type = 'NOT_FOUND' + container = Container(name='foo_bar_container', extra={}, driver=self) + try: + self.driver.delete_container(container=container) + except ContainerDoesNotExistError: + pass + else: + self.fail( + 'Container does not exist but an exception was not thrown') + + def test_delete_container_not_empty(self): + SwiftMockHttp.type = 'NOT_EMPTY' + container = Container(name='foo_bar_container', extra={}, driver=self) + try: + self.driver.delete_container(container=container) + except ContainerIsNotEmptyError: + pass + else: + self.fail('Container is not empty but an exception was not thrown') + + def test_download_object_success(self): + container = Container(name='foo_bar_container', extra={}, driver=self) + obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, + container=container, meta_data=None, + driver=SwiftStorageDriver) + destination_path = os.path.abspath(__file__) + '.temp' + result = self.driver.download_object(obj=obj, + destination_path=destination_path, + overwrite_existing=False, + delete_on_failure=True) + self.assertTrue(result) + + def test_download_object_invalid_file_size(self): + SwiftMockRawResponse.type = 'INVALID_SIZE' + container = Container(name='foo_bar_container', extra={}, driver=self) + obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, + container=container, meta_data=None, + driver=SwiftStorageDriver) + destination_path = os.path.abspath(__file__) + '.temp' + result = self.driver.download_object(obj=obj, + destination_path=destination_path, + overwrite_existing=False, + delete_on_failure=True) + self.assertFalse(result) + + def test_download_object_success_not_found(self): + SwiftMockRawResponse.type = 'NOT_FOUND' + container = Container(name='foo_bar_container', extra={}, driver=self) + + obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, + container=container, + meta_data=None, + driver=SwiftStorageDriver) + destination_path = os.path.abspath(__file__) + '.temp' + try: + self.driver.download_object( + obj=obj, + destination_path=destination_path, + overwrite_existing=False, + delete_on_failure=True) + except ObjectDoesNotExistError: + pass + else: + self.fail('Object does not exist but an exception was not thrown') + + def test_download_object_as_stream(self): + container = Container(name='foo_bar_container', extra={}, driver=self) + obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, + container=container, meta_data=None, + driver=SwiftStorageDriver) + + stream = self.driver.download_object_as_stream(obj=obj, chunk_size=None) + self.assertTrue(hasattr(stream, '__iter__')) + + def test_upload_object_success(self): + def upload_file(self, response, file_path, chunked=False, + calculate_hash=True): + return True, 'hash343hhash89h932439jsaa89', 1000 + + old_func = SwiftStorageDriver._upload_file + SwiftStorageDriver._upload_file = upload_file + file_path = os.path.abspath(__file__) + container = Container(name='foo_bar_container', extra={}, driver=self) + object_name = 'foo_test_upload' + extra = {'meta_data': { 'some-value': 'foobar'}} + obj = self.driver.upload_object(file_path=file_path, container=container, + extra=extra, object_name=object_name) + self.assertEqual(obj.name, 'foo_test_upload') + self.assertEqual(obj.size, 1000) + self.assertTrue('some-value' in obj.meta_data) + SwiftStorageDriver._upload_file = old_func + + def test_upload_object_invalid_hash(self): + def upload_file(self, response, file_path, chunked=False, + calculate_hash=True): + return True, 'hash343hhash89h932439jsaa89', 1000 + + SwiftMockRawResponse.type = 'INVALID_HASH' + + old_func = SwiftStorageDriver._upload_file + SwiftStorageDriver._upload_file = upload_file + file_path = os.path.abspath(__file__) + container = Container(name='foo_bar_container', extra={}, driver=self) + object_name = 'foo_test_upload' + try: + self.driver.upload_object(file_path=file_path, container=container, + object_name=object_name, + verify_hash=True) + except ObjectHashMismatchError: + pass + else: + self.fail( + 'Invalid hash was returned but an exception was not thrown') + finally: + SwiftStorageDriver._upload_file = old_func + + def test_upload_object_no_content_type(self): + def no_content_type(name): + return None, None + + old_func = libcloud.utils.guess_file_mime_type + libcloud.utils.guess_file_mime_type = no_content_type + file_path = os.path.abspath(__file__) + container = Container(name='foo_bar_container', extra={}, driver=self) + object_name = 'foo_test_upload' + try: + self.driver.upload_object(file_path=file_path, container=container, + object_name=object_name) + except AttributeError: + pass + else: + self.fail( + 'File content type not provided' + ' but an exception was not thrown') + finally: + libcloud.utils.guess_file_mime_type = old_func + + def test_upload_object_error(self): + def dummy_content_type(name): + return 'application/zip', None + + def send(instance): + raise Exception('') + + old_func1 = libcloud.utils.guess_file_mime_type + libcloud.utils.guess_file_mime_type = dummy_content_type + old_func2 = SwiftMockHttp.send + SwiftMockHttp.send = send + + file_path = os.path.abspath(__file__) + container = Container(name='foo_bar_container', extra={}, driver=self) + object_name = 'foo_test_upload' + try: + self.driver.upload_object( + file_path=file_path, + container=container, + object_name=object_name) + except LibcloudError: + pass + else: + self.fail('Timeout while uploading but an exception was not thrown') + finally: + libcloud.utils.guess_file_mime_type = old_func1 + SwiftMockHttp.send = old_func2 + + def test_upload_object_inexistent_file(self): + def dummy_content_type(name): + return 'application/zip', None + + old_func = libcloud.utils.guess_file_mime_type + libcloud.utils.guess_file_mime_type = dummy_content_type + + file_path = os.path.abspath(__file__ + '.inexistent') + container = Container(name='foo_bar_container', extra={}, driver=self) + object_name = 'foo_test_upload' + try: + self.driver.upload_object( + file_path=file_path, + container=container, + object_name=object_name) + except OSError: + pass + else: + self.fail('Inesitent but an exception was not thrown') + finally: + libcloud.utils.guess_file_mime_type = old_func + + def test_upload_object_via_stream(self): + def dummy_content_type(name): + return 'application/zip', None + + old_func = libcloud.utils.guess_file_mime_type + libcloud.utils.guess_file_mime_type = dummy_content_type + + container = Container(name='foo_bar_container', extra={}, driver=self) + object_name = 'foo_test_stream_data' + iterator = DummyIterator(data=['2', '3', '5']) + try: + self.driver.upload_object_via_stream(container=container, + object_name=object_name, + iterator=iterator) + finally: + libcloud.utils.guess_file_mime_type = old_func + + def test_delete_object_success(self): + container = Container(name='foo_bar_container', extra={}, driver=self) + obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, + container=container, meta_data=None, + driver=SwiftStorageDriver) + status = self.driver.delete_object(obj=obj) + self.assertTrue(status) + + def test_delete_object_not_found(self): + SwiftMockHttp.type = 'NOT_FOUND' + container = Container(name='foo_bar_container', extra={}, driver=self) + obj = Object(name='foo_bar_object', size=1000, hash=None, extra={}, + container=container, meta_data=None, + driver=SwiftStorageDriver) + try: + self.driver.delete_object(obj=obj) + except ObjectDoesNotExistError: + pass + else: + self.fail('Object does not exist but an exception was not thrown') + + def test_ex_get_meta_data(self): + meta_data = self.driver.ex_get_meta_data() + self.assertTrue(isinstance(meta_data, dict)) + self.assertTrue('object_count' in meta_data) + self.assertTrue('container_count' in meta_data) + self.assertTrue('bytes_used' in meta_data) + + def _remove_test_file(self): + file_path = os.path.abspath(__file__) + '.temp' + + try: + os.unlink(file_path) + except OSError: + pass + +class SwiftMockHttp(StorageMockHttp): + + fixtures = StorageFileFixtures('swift') + base_headers = { 'content-type': 'application/json; charset=UTF-8'} + + # fake auth token response + def _v1_0(self, method, url, body, headers): + headers = copy.deepcopy(self.base_headers) + headers.update({ 'x-auth-token': 'FE011C19', + 'x-storage-token': 'FE011C19', + 'x-storage-url': + 'https://storage4.clouddrive.com/v1/MossoCloudFS'}) + return (httplib.NO_CONTENT, + "", + headers, + httplib.responses[httplib.NO_CONTENT]) + + def _v1_MossoCloudFS_MALFORMED_JSON(self, method, url, body, headers): + # test_invalid_json_throws_exception + body = 'broken: json /*"' + return (httplib.NO_CONTENT, + body, + self.base_headers, + httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_EMPTY(self, method, url, body, headers): + return (httplib.NO_CONTENT, + body, + self.base_headers, + httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS(self, method, url, body, headers): + headers = copy.deepcopy(self.base_headers) + if method == 'GET': + # list_containers + body = self.fixtures.load('list_containers.json') + status_code = httplib.OK + elif method == 'HEAD': + # get_meta_data + body = self.fixtures.load('meta_data.json') + status_code = httplib.NO_CONTENT + headers.update({ 'x-account-container-count': 10, + 'x-account-object-count': 400, + 'x-account-bytes-used': 1234567 + }) + return (status_code, body, headers, httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_not_found(self, method, url, body, headers): + # test_get_object_not_found + if method == 'HEAD': + body = '' + else: + raise ValueError('Invalid method') + + return (httplib.NOT_FOUND, + body, + self.base_headers, + httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_test_container_EMPTY(self, method, url, body, headers): + body = self.fixtures.load('list_container_objects_empty.json') + return (httplib.OK, + body, + self.base_headers, + httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_test_container(self, method, url, body, headers): + headers = copy.deepcopy(self.base_headers) + if method == 'GET': + # list_container_objects + body = self.fixtures.load('list_container_objects.json') + status_code = httplib.OK + elif method == 'HEAD': + # get_container + body = self.fixtures.load('list_container_objects_empty.json') + status_code = httplib.NO_CONTENT + headers.update({ 'x-container-object-count': 800, + 'x-container-bytes-used': 1234568 + }) + return (status_code, body, headers, httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_test_container_not_found( + self, method, url, body, headers): + # test_get_container_not_found + if method == 'HEAD': + body = '' + else: + raise ValueError('Invalid method') + + return (httplib.NOT_FOUND, body, + self.base_headers, + httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_test_container_test_object( + self, method, url, body, headers): + headers = copy.deepcopy(self.base_headers) + if method == 'HEAD': + # get_object + body = self.fixtures.load('list_container_objects_empty.json') + status_code = httplib.NO_CONTENT + headers.update({ 'content-length': 555, + 'last-modified': 'Tue, 25 Jan 2011 22:01:49 GMT', + 'etag': '6b21c4a111ac178feacf9ec9d0c71f17', + 'x-object-meta-foo-bar': 'test 1', + 'x-object-meta-bar-foo': 'test 2', + 'content-type': 'application/zip'}) + return (status_code, body, headers, httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_test_create_container( + self, method, url, body, headers): + # test_create_container_success + headers = copy.deepcopy(self.base_headers) + body = self.fixtures.load('list_container_objects_empty.json') + headers = copy.deepcopy(self.base_headers) + headers.update({ 'content-length': 18, + 'date': 'Mon, 28 Feb 2011 07:52:57 GMT' + }) + status_code = httplib.CREATED + return (status_code, body, headers, httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_test_create_container_ALREADY_EXISTS( + self, method, url, body, headers): + # test_create_container_already_exists + headers = copy.deepcopy(self.base_headers) + body = self.fixtures.load('list_container_objects_empty.json') + headers.update({ 'content-type': 'text/plain' }) + status_code = httplib.ACCEPTED + return (status_code, body, headers, httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_foo_bar_container(self, method, url, body, headers): + if method == 'DELETE': + # test_delete_container_success + body = self.fixtures.load('list_container_objects_empty.json') + headers = self.base_headers + status_code = httplib.NO_CONTENT + return (status_code, body, headers, httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_foo_bar_container_NOT_FOUND( + self, method, url, body, headers): + + if method == 'DELETE': + # test_delete_container_not_found + body = self.fixtures.load('list_container_objects_empty.json') + headers = self.base_headers + status_code = httplib.NOT_FOUND + return (status_code, body, headers, httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_foo_bar_container_NOT_EMPTY( + self, method, url, body, headers): + + if method == 'DELETE': + # test_delete_container_not_empty + body = self.fixtures.load('list_container_objects_empty.json') + headers = self.base_headers + status_code = httplib.CONFLICT + return (status_code, body, headers, httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_foo_bar_container_foo_bar_object( + self, method, url, body, headers): + + if method == 'DELETE': + # test_delete_object_success + body = self.fixtures.load('list_container_objects_empty.json') + headers = self.base_headers + status_code = httplib.NO_CONTENT + return (status_code, body, headers, httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_foo_bar_container_foo_bar_object_NOT_FOUND( + self, method, url, body, headers): + + if method == 'DELETE': + # test_delete_object_success + body = self.fixtures.load('list_container_objects_empty.json') + headers = self.base_headers + status_code = httplib.NOT_FOUND + + return (status_code, body, headers, httplib.responses[httplib.OK]) + +class SwiftMockRawResponse(MockRawResponse): + + fixtures = StorageFileFixtures('swift') + base_headers = { 'content-type': 'application/json; charset=UTF-8'} + + def _v1_MossoCloudFS_foo_bar_container_foo_test_upload( + self, method, url, body, headers): + # test_object_upload_success + + body = '' + headers = {} + headers.update(self.base_headers) + headers['etag'] = 'hash343hhash89h932439jsaa89' + return (httplib.CREATED, body, headers, httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_foo_bar_container_foo_test_upload_INVALID_HASH( + self, method, url, body, headers): + # test_object_upload_invalid_hash + body = '' + headers = {} + headers.update(self.base_headers) + headers['etag'] = 'foobar' + return (httplib.CREATED, body, headers, + httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_foo_bar_container_foo_bar_object( + self, method, url, body, headers): + + # test_download_object_success + body = 'test' + self._data = self._generate_random_data(1000) + return (httplib.OK, + body, + self.base_headers, + httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_foo_bar_container_foo_bar_object_INVALID_SIZE( + self, method, url, body, headers): + # test_download_object_invalid_file_size + body = 'test' + self._data = self._generate_random_data(100) + return (httplib.OK, body, + self.base_headers, + httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_foo_bar_container_foo_bar_object_NOT_FOUND( + self, method, url, body, headers): + body = '' + return (httplib.NOT_FOUND, body, + self.base_headers, + httplib.responses[httplib.OK]) + + def _v1_MossoCloudFS_foo_bar_container_foo_test_stream_data( + self, method, url, body, headers): + + # test_upload_object_via_stream_success + headers = {} + headers.update(self.base_headers) + headers['etag'] = '577ef1154f3240ad5b9b413aa7346a1e' + body = 'test' + return (httplib.CREATED, + body, + headers, + httplib.responses[httplib.OK]) + +if __name__ == '__main__': + sys.exit(unittest.main())