Skip to content

Commit 57abde0

Browse files
jingczhangmandre
jingczhang
authored and
mandre
committed
Get IP addresses of neutron subports (kubernetes#2306)
Vlan-aware VMs are commonly used in telecom. Those VMs are plugged into flat networks and use neutron trunk. https://docs.openstack.org/neutron/latest/admin/config-trunking.html Currenlty, nodeAddresses() only returns neutron ports directly attached to VM. For vlan-aware VMs, IP addresses are assigned on neutron subports. Subports are attached to the trunk; they are not attached to the VM directly. This pull request changes nodeAddresses() to return IP addresses of neutron subports when they exist. Without this change, Kubernetes is unable to IP addresses of vlan-aware VMs. This change is transparent to VMs not using neturon trunk. Signed-off-by: Jing Zhang <jing.c.zhang@nokia.com>
1 parent e41c7e7 commit 57abde0

File tree

4 files changed

+98
-39
lines changed

4 files changed

+98
-39
lines changed

‎pkg/openstack/instances.go‎

Lines changed: 54 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,8 @@ import (
2929
"github.com/gophercloud/gophercloud"
3030
"github.com/gophercloud/gophercloud/openstack/compute/v2/flavors"
3131
"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
32-
"github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
32+
"github.com/gophercloud/gophercloud/openstack/networking/v2/networks"
33+
neutronports "github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
3334
"github.com/gophercloud/gophercloud/pagination"
3435
"github.com/mitchellh/mapstructure"
3536
v1 "k8s.io/api/core/v1"
@@ -43,7 +44,6 @@ import (
4344
"k8s.io/cloud-provider-openstack/pkg/util"
4445
"k8s.io/cloud-provider-openstack/pkg/util/errors"
4546
"k8s.io/cloud-provider-openstack/pkg/util/metadata"
46-
"k8s.io/cloud-provider-openstack/pkg/util/openstack"
4747
)
4848

4949
// Instances encapsulates an implementation of Instances for OpenStack.
@@ -240,7 +240,7 @@ func (i *Instances) NodeAddressesByProviderID(ctx context.Context, providerID st
240240
return []v1.NodeAddress{}, err
241241
}
242242

243-
addresses, err := nodeAddresses(server, ports, i.networkingOpts)
243+
addresses, err := nodeAddresses(server, ports, i.network, i.networkingOpts)
244244
if err != nil {
245245
return []v1.NodeAddress{}, err
246246
}
@@ -345,7 +345,7 @@ func (i *Instances) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloud
345345
if err != nil {
346346
return nil, err
347347
}
348-
addresses, err := nodeAddresses(srv, ports, i.networkingOpts)
348+
addresses, err := nodeAddresses(srv, ports, i.network, i.networkingOpts)
349349
if err != nil {
350350
return nil, err
351351
}
@@ -573,7 +573,7 @@ func getServerByName(client *gophercloud.ServiceClient, name types.NodeName) (*S
573573
// * access IPs
574574
// * metadata hostname
575575
// * server object Addresses (floating type)
576-
func nodeAddresses(srv *servers.Server, ports []ports.Port, networkingOpts NetworkingOpts) ([]v1.NodeAddress, error) {
576+
func nodeAddresses(srv *servers.Server, ports []PortWithTrunkDetails, client *gophercloud.ServiceClient, networkingOpts NetworkingOpts) ([]v1.NodeAddress, error) {
577577
addrs := []v1.NodeAddress{}
578578

579579
// parse private IP addresses first in an ordered manner
@@ -633,6 +633,39 @@ func nodeAddresses(srv *servers.Server, ports []ports.Port, networkingOpts Netwo
633633
return nil, err
634634
}
635635

636+
// Add the addresses assigned on subports via trunk
637+
// This exposes the vlan networks to which subports are attached
638+
for _, port := range ports {
639+
for _, subport := range port.TrunkDetails.SubPorts {
640+
p, err := neutronports.Get(client, subport.PortID).Extract()
641+
if err != nil {
642+
klog.Errorf("Failed to get subport %s details: %v", subport.PortID, err)
643+
continue
644+
}
645+
n, err := networks.Get(client, p.NetworkID).Extract()
646+
if err != nil {
647+
klog.Errorf("Failed to get subport %s network details: %v", subport.PortID, err)
648+
continue
649+
}
650+
for _, fixedIP := range p.FixedIPs {
651+
klog.V(5).Infof("Node '%s' is found subport '%s' address '%s/%s'", srv.Name, p.Name, n.Name, fixedIP.IPAddress)
652+
isIPv6 := net.ParseIP(fixedIP.IPAddress).To4() == nil
653+
if !(isIPv6 && networkingOpts.IPv6SupportDisabled) {
654+
addr := Address{IPType: "fixed", Addr: fixedIP.IPAddress}
655+
subportAddresses := map[string][]Address{n.Name: {addr}}
656+
srvAddresses, ok := addresses[n.Name]
657+
if !ok {
658+
addresses[n.Name] = subportAddresses[n.Name]
659+
} else {
660+
// this is to take care the corner case
661+
// where the same network is attached to the node both directly and via trunk
662+
addresses[n.Name] = append(srvAddresses, subportAddresses[n.Name]...)
663+
}
664+
}
665+
}
666+
}
667+
}
668+
636669
networks := make([]string, 0, len(addresses))
637670
for k := range addresses {
638671
networks = append(networks, k)
@@ -683,6 +716,7 @@ func nodeAddresses(srv *servers.Server, ports []ports.Port, networkingOpts Netwo
683716
sortNodeAddresses(addrs, networkingOpts.AddressSortOrder)
684717
}
685718

719+
klog.V(5).Infof("Node '%s' returns addresses '%v'", srv.Name, addrs)
686720
return addrs, nil
687721
}
688722

@@ -697,14 +731,25 @@ func getAddressesByName(client *gophercloud.ServiceClient, name types.NodeName,
697731
return nil, err
698732
}
699733

700-
return nodeAddresses(&srv.Server, ports, networkingOpts)
734+
return nodeAddresses(&srv.Server, ports, client, networkingOpts)
701735
}
702736

703737
// getAttachedPorts returns a list of ports attached to a server.
704-
func getAttachedPorts(client *gophercloud.ServiceClient, serverID string) ([]ports.Port, error) {
705-
listOpts := ports.ListOpts{
738+
func getAttachedPorts(client *gophercloud.ServiceClient, serverID string) ([]PortWithTrunkDetails, error) {
739+
listOpts := neutronports.ListOpts{
706740
DeviceID: serverID,
707741
}
708742

709-
return openstack.GetPorts(client, listOpts)
743+
var ports []PortWithTrunkDetails
744+
745+
allPages, err := neutronports.List(client, listOpts).AllPages()
746+
if err != nil {
747+
return ports, err
748+
}
749+
err = neutronports.ExtractPortsInto(allPages, &ports)
750+
if err != nil {
751+
return ports, err
752+
}
753+
754+
return ports, nil
710755
}

‎pkg/openstack/instancesv2.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ func (i *InstancesV2) InstanceMetadata(ctx context.Context, node *v1.Node) (*clo
128128
return nil, err
129129
}
130130

131-
addresses, err := nodeAddresses(&server.Server, ports, i.networkingOpts)
131+
addresses, err := nodeAddresses(&server.Server, ports, i.network, i.networkingOpts)
132132
if err != nil {
133133
return nil, err
134134
}

‎pkg/openstack/openstack.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ import (
2727
"github.com/gophercloud/gophercloud"
2828
"github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/availabilityzones"
2929
"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
30+
"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/trunk_details"
31+
neutronports "github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
3032
"github.com/spf13/pflag"
3133
gcfg "gopkg.in/gcfg.v1"
3234
"k8s.io/apimachinery/pkg/types"
@@ -67,6 +69,11 @@ func AddExtraFlags(fs *pflag.FlagSet) {
6769
fs.StringArrayVar(&userAgentData, "user-agent", nil, "Extra data to add to gophercloud user-agent. Use multiple times to add more than one component.")
6870
}
6971

72+
type PortWithTrunkDetails struct {
73+
neutronports.Port
74+
trunk_details.TrunkDetailsExt
75+
}
76+
7077
// LoadBalancer is used for creating and maintaining load balancers
7178
type LoadBalancer struct {
7279
secret *gophercloud.ServiceClient

‎pkg/openstack/openstack_test.go‎

Lines changed: 36 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ import (
2828

2929
"github.com/gophercloud/gophercloud"
3030
"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
31-
"github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
31+
neutronports "github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
3232
"github.com/spf13/pflag"
3333
v1 "k8s.io/api/core/v1"
3434
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -373,10 +373,10 @@ func TestNodeAddresses(t *testing.T) {
373373
PublicNetworkName: []string{"public"},
374374
}
375375

376-
ports := []ports.Port{
377-
{
376+
ports := []PortWithTrunkDetails{{
377+
Port: neutronports.Port{
378378
Status: "ACTIVE",
379-
FixedIPs: []ports.IP{
379+
FixedIPs: []neutronports.IP{
380380
{
381381
IPAddress: "10.0.0.32",
382382
},
@@ -385,9 +385,10 @@ func TestNodeAddresses(t *testing.T) {
385385
},
386386
},
387387
},
388+
},
388389
}
389390

390-
addrs, err := nodeAddresses(&srv, ports, networkingOpts)
391+
addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts)
391392
if err != nil {
392393
t.Fatalf("nodeAddresses returned error: %v", err)
393394
}
@@ -452,10 +453,10 @@ func TestNodeAddressesCustomPublicNetwork(t *testing.T) {
452453
PublicNetworkName: []string{"pub-net"},
453454
}
454455

455-
ports := []ports.Port{
456-
{
456+
ports := []PortWithTrunkDetails{{
457+
Port: neutronports.Port{
457458
Status: "ACTIVE",
458-
FixedIPs: []ports.IP{
459+
FixedIPs: []neutronports.IP{
459460
{
460461
IPAddress: "10.0.0.32",
461462
},
@@ -464,9 +465,10 @@ func TestNodeAddressesCustomPublicNetwork(t *testing.T) {
464465
},
465466
},
466467
},
468+
},
467469
}
468470

469-
addrs, err := nodeAddresses(&srv, ports, networkingOpts)
471+
addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts)
470472
if err != nil {
471473
t.Fatalf("nodeAddresses returned error: %v", err)
472474
}
@@ -525,10 +527,10 @@ func TestNodeAddressesCustomPublicNetworkWithIntersectingFixedIP(t *testing.T) {
525527
PublicNetworkName: []string{"pub-net"},
526528
}
527529

528-
ports := []ports.Port{
529-
{
530+
ports := []PortWithTrunkDetails{{
531+
Port: neutronports.Port{
530532
Status: "ACTIVE",
531-
FixedIPs: []ports.IP{
533+
FixedIPs: []neutronports.IP{
532534
{
533535
IPAddress: "10.0.0.32",
534536
},
@@ -541,9 +543,10 @@ func TestNodeAddressesCustomPublicNetworkWithIntersectingFixedIP(t *testing.T) {
541543
},
542544
},
543545
},
546+
},
544547
}
545548

546-
addrs, err := nodeAddresses(&srv, ports, networkingOpts)
549+
addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts)
547550
if err != nil {
548551
t.Fatalf("nodeAddresses returned error: %v", err)
549552
}
@@ -613,10 +616,10 @@ func TestNodeAddressesMultipleCustomInternalNetworks(t *testing.T) {
613616
InternalNetworkName: []string{"private", "also-private"},
614617
}
615618

616-
ports := []ports.Port{
617-
{
619+
ports := []PortWithTrunkDetails{{
620+
Port: neutronports.Port{
618621
Status: "ACTIVE",
619-
FixedIPs: []ports.IP{
622+
FixedIPs: []neutronports.IP{
620623
{
621624
IPAddress: "10.0.0.32",
622625
},
@@ -625,9 +628,10 @@ func TestNodeAddressesMultipleCustomInternalNetworks(t *testing.T) {
625628
},
626629
},
627630
},
631+
},
628632
}
629633

630-
addrs, err := nodeAddresses(&srv, ports, networkingOpts)
634+
addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts)
631635
if err != nil {
632636
t.Fatalf("nodeAddresses returned error: %v", err)
633637
}
@@ -697,10 +701,10 @@ func TestNodeAddressesOneInternalNetwork(t *testing.T) {
697701
InternalNetworkName: []string{"also-private"},
698702
}
699703

700-
ports := []ports.Port{
701-
{
704+
ports := []PortWithTrunkDetails{{
705+
Port: neutronports.Port{
702706
Status: "ACTIVE",
703-
FixedIPs: []ports.IP{
707+
FixedIPs: []neutronports.IP{
704708
{
705709
IPAddress: "10.0.0.32",
706710
},
@@ -709,9 +713,10 @@ func TestNodeAddressesOneInternalNetwork(t *testing.T) {
709713
},
710714
},
711715
},
716+
},
712717
}
713718

714-
addrs, err := nodeAddresses(&srv, ports, networkingOpts)
719+
addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts)
715720
if err != nil {
716721
t.Fatalf("nodeAddresses returned error: %v", err)
717722
}
@@ -773,10 +778,10 @@ func TestNodeAddressesIPv6Disabled(t *testing.T) {
773778
IPv6SupportDisabled: true,
774779
}
775780

776-
ports := []ports.Port{
777-
{
781+
ports := []PortWithTrunkDetails{{
782+
Port: neutronports.Port{
778783
Status: "ACTIVE",
779-
FixedIPs: []ports.IP{
784+
FixedIPs: []neutronports.IP{
780785
{
781786
IPAddress: "10.0.0.32",
782787
},
@@ -785,9 +790,10 @@ func TestNodeAddressesIPv6Disabled(t *testing.T) {
785790
},
786791
},
787792
},
793+
},
788794
}
789795

790-
addrs, err := nodeAddresses(&srv, ports, networkingOpts)
796+
addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts)
791797
if err != nil {
792798
t.Fatalf("nodeAddresses returned error: %v", err)
793799
}
@@ -854,10 +860,10 @@ func TestNodeAddressesWithAddressSortOrderOptions(t *testing.T) {
854860
AddressSortOrder: "10.0.0.0/8, 50.56.176.0/24, 2001:4800::/32",
855861
}
856862

857-
ports := []ports.Port{
858-
{
863+
ports := []PortWithTrunkDetails{{
864+
Port: neutronports.Port{
859865
Status: "ACTIVE",
860-
FixedIPs: []ports.IP{
866+
FixedIPs: []neutronports.IP{
861867
{
862868
IPAddress: "10.0.0.32",
863869
},
@@ -866,9 +872,10 @@ func TestNodeAddressesWithAddressSortOrderOptions(t *testing.T) {
866872
},
867873
},
868874
},
875+
},
869876
}
870877

871-
addrs, err := nodeAddresses(&srv, ports, networkingOpts)
878+
addrs, err := nodeAddresses(&srv, ports, nil, networkingOpts)
872879
if err != nil {
873880
t.Fatalf("nodeAddresses returned error: %v", err)
874881
}

0 commit comments

Comments
 (0)