k8s_info openshift.dynamic.exceptions.ServiceUnavailableError: 503

[wmlynch]

Target cluster version: Openshift 4.2.5
openshift pip version: 0.10.0
kubernetes pip version: 10.0.1
ansible 2.9.2

- name: wait for kubernetes service
  k8s_info:
    api_version: v1
    kind: Service
    kubeconfig: "{{ cluster_kubeconfig }}"
    name: kubernetes
    namespace: default
  register: _kube_service

Ansible std_error block:

    "stderr_lines": [
        "Traceback (most recent call last):",
        "  File \"/usr/lib/python3.7/site-packages/openshift/dynamic/client.py\", line 42, in inner",
        "    resp = func(self, *args, **kwargs)",
        "  File \"/usr/lib/python3.7/site-packages/openshift/dynamic/client.py\", line 241, in request",
        "    _return_http_data_only=params.get('_return_http_data_only', True)",
        "  File \"/usr/lib/python3.7/site-packages/kubernetes/client/api_client.py\", line 334, in call_api",
        "    _return_http_data_only, collection_formats, _preload_content, _request_timeout)",
        "  File \"/usr/lib/python3.7/site-packages/kubernetes/client/api_client.py\", line 168, in __call_api",
        "    _request_timeout=_request_timeout)",
        "  File \"/usr/lib/python3.7/site-packages/kubernetes/client/api_client.py\", line 355, in request",
        "    headers=headers)",
        "  File \"/usr/lib/python3.7/site-packages/kubernetes/client/rest.py\", line 231, in GET",
        "    query_params=query_params)",
        "  File \"/usr/lib/python3.7/site-packages/kubernetes/client/rest.py\", line 222, in request",
        "    raise ApiException(http_resp=r)",
        "kubernetes.client.rest.ApiException: (503)",
        "Reason: Service Unavailable",
        "HTTP response headers: HTTPHeaderDict({'Cache-Control': 'no-cache, private', 'Content-Type': 'text/plain; charset=utf-8', 'X-Content-Type-Options': 'nosniff', 'Date': 'Thu, 12 Dec 2019 23:00:40 GMT', 'Content-Length': '20'})",
        "HTTP response body: b'service unavailable\\n'",
        "",
        "",
        "During handling of the above exception, another exception occurred:",
        "",
        "Traceback (most recent call last):",
        "  File \"/root/.ansible/tmp/ansible-tmp-1576191621.8594584-53701596979993/AnsiballZ_k8s_info.py\", line 102, in <module>",
        "    _ansiballz_main()",
        "  File \"/root/.ansible/tmp/ansible-tmp-1576191621.8594584-53701596979993/AnsiballZ_k8s_info.py\", line 94, in _ansiballz_main",
        "    invoke_module(zipped_mod, temp_path, ANSIBALLZ_PARAMS)",
        "  File \"/root/.ansible/tmp/ansible-tmp-1576191621.8594584-53701596979993/AnsiballZ_k8s_info.py\", line 40, in invoke_module",
        "    runpy.run_module(mod_name='ansible.modules.clustering.k8s.k8s_info', init_globals=None, run_name='__main__', alter_sys=True)",
        "  File \"/usr/lib/python3.7/runpy.py\", line 205, in run_module",
        "    return _run_module_code(code, init_globals, run_name, mod_spec)",
        "  File \"/usr/lib/python3.7/runpy.py\", line 96, in _run_module_code",
        "    mod_name, mod_spec, pkg_name, script_name)",
        "  File \"/usr/lib/python3.7/runpy.py\", line 85, in _run_code",
        "    exec(code, run_globals)",
        "  File \"/tmp/ansible_k8s_info_payload_hb4to5hf/ansible_k8s_info_payload.zip/ansible/modules/clustering/k8s/k8s_info.py\", line 179, in <module>",
        "  File \"/tmp/ansible_k8s_info_payload_hb4to5hf/ansible_k8s_info_payload.zip/ansible/modules/clustering/k8s/k8s_info.py\", line 175, in main",
        "  File \"/tmp/ansible_k8s_info_payload_hb4to5hf/ansible_k8s_info_payload.zip/ansible/modules/clustering/k8s/k8s_info.py\", line 156, in execute_module",
        "  File \"/tmp/ansible_k8s_info_payload_hb4to5hf/ansible_k8s_info_payload.zip/ansible/module_utils/k8s/common.py\", line 215, in kubernetes_facts",
        "  File \"/tmp/ansible_k8s_info_payload_hb4to5hf/ansible_k8s_info_payload.zip/ansible/module_utils/k8s/common.py\", line 205, in find_resource",
        "  File \"/usr/lib/python3.7/site-packages/openshift/dynamic/discovery.py\", line 201, in get",
        "    results = self.search(**kwargs)",
        "  File \"/usr/lib/python3.7/site-packages/openshift/dynamic/discovery.py\", line 242, in search",
        "    results = self.__search(self.__build_search(**kwargs), self.__resources, [])",
        "  File \"/usr/lib/python3.7/site-packages/openshift/dynamic/discovery.py\", line 284, in __search",
        "    matches.extend(self.__search([key] + parts[1:], resources, reqParams))",
        "  File \"/usr/lib/python3.7/site-packages/openshift/dynamic/discovery.py\", line 270, in __search",
        "    return self.__search(parts[1:], resourcePart, reqParams + [part] )",
        "  File \"/usr/lib/python3.7/site-packages/openshift/dynamic/discovery.py\", line 284, in __search",
        "    matches.extend(self.__search([key] + parts[1:], resources, reqParams))",
        "  File \"/usr/lib/python3.7/site-packages/openshift/dynamic/discovery.py\", line 270, in __search",
        "    return self.__search(parts[1:], resourcePart, reqParams + [part] )",
        "  File \"/usr/lib/python3.7/site-packages/openshift/dynamic/discovery.py\", line 263, in __search",
        "    group, part, resourcePart.preferred)",
        "  File \"/usr/lib/python3.7/site-packages/openshift/dynamic/discovery.py\", line 166, in get_resources_for_api_version",
        "    resources_response = self.client.request('GET', path).resources or []",
        "  File \"/usr/lib/python3.7/site-packages/openshift/dynamic/client.py\", line 44, in inner",
        "    raise api_exception(e)",
        "openshift.dynamic.exceptions.ServiceUnavailableError: 503",
        "Reason: Service Unavailable",
        "HTTP response headers: HTTPHeaderDict({'Cache-Control': 'no-cache, private', 'Content-Type': 'text/plain; charset=utf-8', 'X-Content-Type-Options': 'nosniff', 'Date': 'Thu, 12 Dec 2019 23:00:40 GMT', 'Content-Length': '20'})",
        "HTTP response body: b'service unavailable\\n'",
        "Original traceback: ",
        "  File \"/usr/lib/python3.7/site-packages/openshift/dynamic/client.py\", line 42, in inner",
        "    resp = func(self, *args, **kwargs)",
        "",
        "  File \"/usr/lib/python3.7/site-packages/openshift/dynamic/client.py\", line 241, in request",
        "    _return_http_data_only=params.get('_return_http_data_only', True)",
        "",
        "  File \"/usr/lib/python3.7/site-packages/kubernetes/client/api_client.py\", line 334, in call_api",
        "    _return_http_data_only, collection_formats, _preload_content, _request_timeout)",
        "",
        "  File \"/usr/lib/python3.7/site-packages/kubernetes/client/api_client.py\", line 168, in __call_api",
        "    _request_timeout=_request_timeout)",
        "",
        "  File \"/usr/lib/python3.7/site-packages/kubernetes/client/api_client.py\", line 355, in request",
        "    headers=headers)",
        "",
        "  File \"/usr/lib/python3.7/site-packages/kubernetes/client/rest.py\", line 231, in GET",
        "    query_params=query_params)",
        "",
        "  File \"/usr/lib/python3.7/site-packages/kubernetes/client/rest.py\", line 222, in request",
        "    raise ApiException(http_resp=r)",
        ""
    ]

Inspecting the cluster, post k8s_info failure, the following apiservices are in a failed state.

apiservice.apiregistration.k8s.io/v1.packages.operators.coreos.com         openshift-operator-lifecycle-manager/v1-packages-operators-coreos-com   False (FailedDiscoveryCheck) 
apiservice.apiregistration.k8s.io/v1beta1.metrics.k8s.io                   openshift-monitoring/prometheus-adapter                                 False (FailedDiscoveryCheck) 

I'm surmising that one or both of the apiservices in FailedDiscoveryCheck caused the failure. What I'm struggling to understand is why.

Some questions/observations:

• Based on discussions in issue Client init fails if one of apis is unavailable #327 the openshift client is gathering all v1 resources, Is this correct?
• If so, why?
• I suspect that if I called oc get service -n default kubernetes -o yaml the command would work successfully.

[fabianvf]

Hmm, I'm not sure why it would be trying to hit the endpoints that are down, unless discovery as a whole is broken because of it. It should be lazily accessing endpoints as needed,

Some questions/observations:

• Based on discussions in issue Client init fails if one of apis is unavailable #327 the openshift client is gathering all v1 resources, Is this correct?

Yes, we're discovering all the core V1 resources (ie, /api/v1)

• If so, why?

The client is fully dynamic, and uses the discovery interface to discover all resources, including the core resources. This lets us handle it all very generically without needing any special casing for core resources.

• I suspect that if I called oc get service -n default kubernetes -o yaml the command would work successfully.

Yes I'd expect that's true, though could you try?

[uditgaurav]

I am also facing a similar problem with
openshift version: 4.2.0
python version is 2.7.12
ansible version is 2.8.0

Error is reporting:

The full traceback is:
Traceback (most recent call last):
  File "/home/devuser/.ansible/tmp/ansible-tmp-1577449654.13-59625285517579/AnsiballZ_k8s_facts.py", line 114, in <module>
    _ansiballz_main()
  File "/home/devuser/.ansible/tmp/ansible-tmp-1577449654.13-59625285517579/AnsiballZ_k8s_facts.py", line 106, in _ansiballz_main
    invoke_module(zipped_mod, temp_path, ANSIBALLZ_PARAMS)
  File "/home/devuser/.ansible/tmp/ansible-tmp-1577449654.13-59625285517579/AnsiballZ_k8s_facts.py", line 49, in invoke_module
    imp.load_module('__main__', mod, module, MOD_DESC)
  File "/tmp/ansible_k8s_facts_payload_Hs0xx_/__main__.py", line 176, in <module>
  File "/tmp/ansible_k8s_facts_payload_Hs0xx_/__main__.py", line 172, in main
  File "/tmp/ansible_k8s_facts_payload_Hs0xx_/__main__.py", line 153, in execute_module
  File "/tmp/ansible_k8s_facts_payload_Hs0xx_/ansible_k8s_facts_payload.zip/ansible/module_utils/k8s/common.py", line 199, in kubernetes_facts
  File "/tmp/ansible_k8s_facts_payload_Hs0xx_/ansible_k8s_facts_payload.zip/ansible/module_utils/k8s/common.py", line 189, in find_resource
  File "/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py", line 227, in get
    results = self.search(**kwargs)
  File "/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py", line 272, in search
    results = self.__search(self.__build_search(**kwargs), self.__resources, [])
  File "/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py", line 314, in __search
    matches.extend(self.__search([key] + parts[1:], resources, reqParams))
  File "/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py", line 300, in __search
    return self.__search(parts[1:], resourcePart, reqParams + [part] )
  File "/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py", line 314, in __search
    matches.extend(self.__search([key] + parts[1:], resources, reqParams))
  File "/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py", line 300, in __search
    return self.__search(parts[1:], resourcePart, reqParams + [part] )
  File "/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py", line 293, in __search
    group, part, resourcePart.preferred)
  File "/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py", line 185, in get_resources_for_api_version
    resources_response = self.client.request('GET', path).resources or []
  File "/usr/local/lib/python2.7/dist-packages/openshift/dynamic/client.py", line 44, in inner
    raise api_exception(e)
openshift.dynamic.exceptions.ServiceUnavailableError: 503
Reason: Service Unavailable
HTTP response headers: HTTPHeaderDict({'Date': 'Fri, 27 Dec 2019 12:27:35 GMT', 'Content-Length': '20', 'Content-Type': 'text/plain; charset=utf-8', 'X-Content-Type-Options': 'nosniff', 'Cache-Control': 'no-cache, private'})
HTTP response body: service unavailable

Original traceback: 
  File "/usr/local/lib/python2.7/dist-packages/openshift/dynamic/client.py", line 42, in inner
    resp = func(self, *args, **kwargs)

  File "/usr/local/lib/python2.7/dist-packages/openshift/dynamic/client.py", line 245, in request
    _return_http_data_only=params.get('_return_http_data_only', True)

  File "/usr/local/lib/python2.7/dist-packages/kubernetes/client/api_client.py", line 334, in call_api
    _return_http_data_only, collection_formats, _preload_content, _request_timeout)

  File "/usr/local/lib/python2.7/dist-packages/kubernetes/client/api_client.py", line 168, in __call_api
    _request_timeout=_request_timeout)

  File "/usr/local/lib/python2.7/dist-packages/kubernetes/client/api_client.py", line 355, in request
    headers=headers)

  File "/usr/local/lib/python2.7/dist-packages/kubernetes/client/rest.py", line 231, in GET
    query_params=query_params)

  File "/usr/local/lib/python2.7/dist-packages/kubernetes/client/rest.py", line 222, in request
    raise ApiException(http_resp=r)


fatal: [localhost]: FAILED! => {
    "changed": false, 
    "module_stderr": "Traceback (most recent call last):\n  File \"/home/devuser/.ansible/tmp/ansible-tmp-1577449654.13-59625285517579/AnsiballZ_k8s_facts.py\", line 114, in <module>\n    _ansiballz_main()\n  File \"/home/devuser/.ansible/tmp/ansible-tmp-1577449654.13-59625285517579/AnsiballZ_k8s_facts.py\", line 106, in _ansiballz_main\n    invoke_module(zipped_mod, temp_path, ANSIBALLZ_PARAMS)\n  File \"/home/devuser/.ansible/tmp/ansible-tmp-1577449654.13-59625285517579/AnsiballZ_k8s_facts.py\", line 49, in invoke_module\n    imp.load_module('__main__', mod, module, MOD_DESC)\n  File \"/tmp/ansible_k8s_facts_payload_Hs0xx_/__main__.py\", line 176, in <module>\n  File \"/tmp/ansible_k8s_facts_payload_Hs0xx_/__main__.py\", line 172, in main\n  File \"/tmp/ansible_k8s_facts_payload_Hs0xx_/__main__.py\", line 153, in execute_module\n  File \"/tmp/ansible_k8s_facts_payload_Hs0xx_/ansible_k8s_facts_payload.zip/ansible/module_utils/k8s/common.py\", line 199, in kubernetes_facts\n  File \"/tmp/ansible_k8s_facts_payload_Hs0xx_/ansible_k8s_facts_payload.zip/ansible/module_utils/k8s/common.py\", line 189, in find_resource\n  File \"/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py\", line 227, in get\n    results = self.search(**kwargs)\n  File \"/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py\", line 272, in search\n    results = self.__search(self.__build_search(**kwargs), self.__resources, [])\n  File \"/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py\", line 314, in __search\n    matches.extend(self.__search([key] + parts[1:], resources, reqParams))\n  File \"/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py\", line 300, in __search\n    return self.__search(parts[1:], resourcePart, reqParams + [part] )\n  File \"/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py\", line 314, in __search\n    matches.extend(self.__search([key] + parts[1:], resources, reqParams))\n  File \"/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py\", line 300, in __search\n    return self.__search(parts[1:], resourcePart, reqParams + [part] )\n  File \"/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py\", line 293, in __search\n    group, part, resourcePart.preferred)\n  File \"/usr/local/lib/python2.7/dist-packages/openshift/dynamic/discovery.py\", line 185, in get_resources_for_api_version\n    resources_response = self.client.request('GET', path).resources or []\n  File \"/usr/local/lib/python2.7/dist-packages/openshift/dynamic/client.py\", line 44, in inner\n    raise api_exception(e)\nopenshift.dynamic.exceptions.ServiceUnavailableError: 503\nReason: Service Unavailable\nHTTP response headers: HTTPHeaderDict({'Date': 'Fri, 27 Dec 2019 12:27:35 GMT', 'Content-Length': '20', 'Content-Type': 'text/plain; charset=utf-8', 'X-Content-Type-Options': 'nosniff', 'Cache-Control': 'no-cache, private'})\nHTTP response body: service unavailable\n\nOriginal traceback: \n  File \"/usr/local/lib/python2.7/dist-packages/openshift/dynamic/client.py\", line 42, in inner\n    resp = func(self, *args, **kwargs)\n\n  File \"/usr/local/lib/python2.7/dist-packages/openshift/dynamic/client.py\", line 245, in request\n    _return_http_data_only=params.get('_return_http_data_only', True)\n\n  File \"/usr/local/lib/python2.7/dist-packages/kubernetes/client/api_client.py\", line 334, in call_api\n    _return_http_data_only, collection_formats, _preload_content, _request_timeout)\n\n  File \"/usr/local/lib/python2.7/dist-packages/kubernetes/client/api_client.py\", line 168, in __call_api\n    _request_timeout=_request_timeout)\n\n  File \"/usr/local/lib/python2.7/dist-packages/kubernetes/client/api_client.py\", line 355, in request\n    headers=headers)\n\n  File \"/usr/local/lib/python2.7/dist-packages/kubernetes/client/rest.py\", line 231, in GET\n    query_params=query_params)\n\n  File \"/usr/local/lib/python2.7/dist-packages/kubernetes/client/rest.py\", line 222, in request\n    raise ApiException(http_resp=r)\n\n", 
    "module_stdout": "", 
    "msg": "MODULE FAILURE\nSee stdout/stderr for the exact error", 
    "rc": 1
}

To reproduce run below ansible in above version

- hosts: localhost

  tasks:
    - name: Get a list of all pods
      k8s_facts:
        api_version: v1
        kind: Pod 
        namespace: default
      register: pod_list

Is there any solution or workaround to resolve the above issue?

[wmlynch]

@uditgaurav This PR fixes the issue for me: #350