Fix python2->3 port: broken imports, dropped test logic, and real bugs

The test suite could not run at all in this PR: every test file mixed
a bare `from BaseTestCase import X` (needs tests/ on sys.path) with
`from lib import X` (needs the repo root on sys.path), while the
leftover sys.path.insert only added <repo_root>/lib, satisfying
neither. Normalized every test file + BaseTestCase.py to insert the
repo root and import BaseTestCase via `tests.BaseTestCase`. Also gave
lib/processor.py and lib/networks.py package-relative imports for
ipcalc, since bare `import ipcalc` only worked when lib/ itself was on
sys.path (the old, now-abandoned convention).

Real bugs found once the suite could actually run:
- TestParser.py: testParseNetwork/testParseNetworkNoVlan silently lost
  their processor.network(...) setup calls during the port, so they
  asserted against an empty table. Restored them.
- processor.py host(): vlan lookup used the host's own freshly-created
  node_id instead of the passed-in network_id, so vlan (and therefore
  IPv6 address generation) was always None. Also dropped a bogus
  `row[5] = json.dumps(row[5])` that JSON-encoded the network_id
  foreign key before insert.
- processor.py network(): `len(net_ipv4)` crashed (ipcalc.Network has
  no __len__) on every call; and the function returned a
  (node_id, network_id) tuple instead of just node_id, corrupting
  parse()'s network_id tracking for every host/network line after the
  first network definition in a file.
- lib/packages.py, lib/firewall.py: leftover `.iteritems()` calls
  (removed in Python 3). One of them (firewall.py's redundant-flow
  pruning loop) had also lost its `node, services` tuple unpacking,
  silently disabling that logic entirely rather than crashing.
- lib/location.py switch_locations(): `/` now does true division in
  Python 3 where Python 2 silently floor-divided, producing fractional
  pixel coordinates. Restored `//` to keep whole-pixel output.
- lib/ipcalc.py: removed a redundant duplicate `self.mask = int(...)`
  line.
- requirements.txt: removed `requests`, `layout`, `ipcalc`, `six`,
  `processor` (layout/ipcalc/processor are this project's own lib/
  modules, not PyPI packages; requests/six are unused), and bumped
  PyYAML off the ancient 3.11 pin, which fails to build against modern
  CPython.
- makefile: test/coverage/draw targets still invoked python2.7.

TestSeatmap.testAddCoordinates/testSwitchLocation/
testSwitchLocationWithMixedLayout expectations updated for two
inherent, unavoidable Python 2->3 differences that surfaced once the
suite could run: dict iteration order (insertion-order in Py3 vs
hash-order in Py2) changes which hall's switches get inserted first,
and round()'s tie-breaking rule (round-half-to-even in Py3 vs
round-half-away-from-zero in Py2) changes one scaled-width value by 2px.

Verified: full test suite passes under Python 3, and an end-to-end
`generate.py` run against real fixture data produces correct host
vlan/IPv6/network_id data (previously broken by the host() bug above).

Author: SoundGoof

lib/firewall.py

@@ -24,7 +24,7 @@ def __eq__(self, other):
 
 
 def add_services(services, c):
-    for service, data in services.iteritems():
+    for service, data in services.items():
         row = [service,
                data.get('description', service),
                ','.join(data['destport']),
@@ -97,7 +97,7 @@ def prefetch_node_and_services(self):
                         self.register_service(access, node, service)
 
             # Prune redundant flows (hosts that share the network flows)
-            for node in self.node_services[access].iteritems():
+            for node, services in self.node_services[access].items():
                 if node not in self.netmap:
                     continue
                 parent = self.node_services[access].get(self.netmap[node])
@@ -110,7 +110,7 @@ def prefetch_node_and_services(self):
                 self.service_nodes[access][srv].add((node, srv))
 
     def node_service_iter(self, access):
-        for node, services in self.node_services[access].iteritems():
+        for node, services in self.node_services[access].items():
             for service in services:
                 yield (node, service)
 

lib/ipcalc.py

@@ -181,8 +181,6 @@ def __init__(self, ip, mask=None, version=0):
         # Netmask is numeric CIDR subnet
         elif isinstance(self.mask, int) or (isinstance(self.mask, str) and self.mask.isdigit()):
             self.mask = int(self.mask)
-
-            self.mask = int(self.mask)
         # Netmask is in subnet notation
         elif isinstance(self.mask, str):
             limit = [32, 128][':' in self.mask]

lib/location.py

@@ -123,13 +123,13 @@ def switch_locations(t, n):
 
     if t.horizontal:
         for i in range(1, 2 * n, 2):
-            x = t.x_start + (t.width / n) / 2 * i
-            y = t.y_start - t.height / 2
+            x = t.x_start + (t.width // n) // 2 * i
+            y = t.y_start - t.height // 2
             locations.append((x,y))
     else:
         for i in range(1, 2 * n, 2):
-            x = t.x_start - t.height / 2
-            y = t.y_start + (t.width / n) / 2 * i
+            x = t.x_start - t.height // 2
+            y = t.y_start + (t.width // n) // 2 * i
             locations.append((x,y))
 
     return locations

lib/networks.py

@@ -1,4 +1,4 @@
-import ipcalc
+from . import ipcalc
 from .processor import ip2long, node
 
 

lib/packages.py

@@ -8,7 +8,7 @@ def default_packages(packages, os):
     # Avoid duplicates
     emitted = set()
     # Interpret X,Y.. pattern and match against OS
-    for pattern, packages in sorted(defaults.iteritems()):
+    for pattern, packages in sorted(defaults.items()):
         if pattern != 'all':
             if os not in pattern.split(','):
                 continue
@@ -60,7 +60,7 @@ def build(packages, c):
         packmap = nodes[node_id]
         # For hosts include network packages
         if node_id not in networks and netmap[node_id] in nodes:
-            for n, p in nodes[netmap[node_id]].iteritems():
+            for n, p in nodes[netmap[node_id]].items():
                 packmap[n].extend(p)
         # Add "default" to hosts, but not networks
         if '-default' not in packmap and node_id not in networks:
@@ -76,7 +76,7 @@ def build(packages, c):
                 del packmap[package]
 
     for node_id, packmap in nodes.items():
-        for package, options in sorted(packmap.iteritems()):
+        for package, options in sorted(packmap.items()):
             for option in options or [None]:
                 row = [node_id, package, option]
                 logging.debug('%d has package %s with %s option',

lib/processor.py

@@ -1,12 +1,12 @@
-import ipcalc
-import json
 import logging
 import re
 import socket
 import struct
 import sys
 from binascii import hexlify
 
+from . import ipcalc
+
 MODULE = sys.modules[__name__]
 
 SYNTAX = {
@@ -71,10 +71,10 @@ def master_network(l, c, r):
 
 def host(l, c, network_id):
     node_id = node(c)
-    c.execute('''SELECT vlan FROM network WHERE node_id = ?''', (node_id,))
+    c.execute('''SELECT vlan FROM network WHERE node_id = ?''', (network_id,))
     row = c.fetchone()
-    if row != None:
-        vlan = int(row[0])
+    if row is not None:
+        vlan = int(row[0]) if row[0] is not None else None
     else:
         vlan = None
 
@@ -106,7 +106,6 @@ def host(l, c, network_id):
         ipv4_addr,
         ipv6_addr,
         network_id]
-    row[5] = json.dumps(row[5])
     c.execute('INSERT INTO host VALUES (?,?,?,?,?,?)', row)
 
     options(c, node_id, l[3])
@@ -123,10 +122,7 @@ def network(l, c, network_id=None):
     # IPv4
     ipv4 = l[1]
     net_ipv4 = ipcalc.Network(ipv4)
-    if len(net_ipv4) <= 2:
-        ipv4_gateway = net_ipv4[0]
-    else:
-        ipv4_gateway = net_ipv4[1]
+    ipv4_gateway = net_ipv4[1]
     ipv4_netmask = str(net_ipv4.netmask())
     ipv4_netmask_dec = int(str(ipv4).split("/")[1])
 
@@ -152,7 +148,7 @@ def network(l, c, network_id=None):
 
     options(c, node_id, l[4])
 
-    return node_id, network_id
+    return node_id
 
 
 def split_value(string):

makefile

@@ -1,9 +1,9 @@
 test:
-	python2.7 tests/TestPackages.py
-	python2.7 tests/TestParser.py
-	python2.7 tests/TestNetworks.py
-	python2.7 tests/TestFirewall.py
-	python2.7 tests/TestSeatmap.py
+	python3 tests/TestPackages.py
+	python3 tests/TestParser.py
+	python3 tests/TestNetworks.py
+	python3 tests/TestFirewall.py
+	python3 tests/TestSeatmap.py
 
 coverage:
 	coverage erase
@@ -17,7 +17,7 @@ coverage:
 	coverage report -m
 
 draw:
-	python2.7 viewer.py --database ipplan.db --hall D
+	python3 viewer.py --database ipplan.db --hall D
 
 lint:
 	pep8 -r .

requirements.txt

@@ -1,6 +1 @@
-PyYAML==3.11
-requests
-layout
-ipcalc
-six
-processor
+PyYAML>=6.0

tests/BaseTestCase.py

@@ -8,7 +8,7 @@
 
 from collections import namedtuple
 
-path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../lib'))
+path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 sys.path.insert(1, path)
 from lib import tables
 

tests/TestFirewall.py

@@ -1,14 +1,14 @@
 import os
 import sys
 import unittest
-from BaseTestCase import BaseTestCase
 
-path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../lib'))
+path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 sys.path.insert(1, path)
 from lib import firewall
 from lib import networks
 from lib import packages
 from lib import processor
+from tests.BaseTestCase import BaseTestCase
 
 
 class TestFirewall(BaseTestCase, unittest.TestCase):