A train-trip worth of python3 migration and other polishing

bin/varnishtest/huffman_gen.py

@@ -11,7 +11,7 @@ if len(sys.argv) != 2:
     sys.exit(2)
 
 class sym:
-	def __init__(self, bigval, bigvall, chr = 0, esc = None):
+    def __init__(self, bigval, bigvall, chr=0, esc=None):
         self.vall = bigvall % 8 if bigvall % 8 else 8
         self.val = bigval & ((1 << self.vall) - 1)
         self.pfx = (bigval >> self.vall)# & 0xff
@@ -28,7 +28,7 @@ for l in f:
     if not match:
         continue
 
-	chr = int(match.group(grp), 16)
+    char = int(match.group(grp), 16)
     grp += 1
 
     val = int(match.group(grp), 16)
@@ -36,12 +36,12 @@ for l in f:
 
     vall = int(match.group(grp))
 
-	s = sym(val, vall, chr)
+    s = sym(val, vall, char)
     if s.pfx not in tbls:
         tbls[s.pfx] = {}
 
-	if (s.val in tbls[s.pfx]):
-		assert(tbls[s.pfx][s.val].e)
+    if s.val in tbls[s.pfx]:
+        assert tbls[s.pfx][s.val].e
     tbls[s.pfx][s.val] = s
 
     # add the escape entry in the "previous" table
@@ -76,15 +76,15 @@ struct stbl {
 ''')
 
 for pfx in sorted(tbls.keys(), reverse=True):
-	msl = max([ x.vall for x in tbls[pfx].values() ])
+    msl = max([x.vall for x in tbls[pfx].values()])
     for s in tbls[pfx].values():
         s.val = s.val << (msl - s.vall)
 
-	tbl = sorted(tbls[pfx].values(), key= lambda x: x.val)
+    tbl = sorted(tbls[pfx].values(), key=lambda x: x.val)
     print("\nstatic struct ssym sym_{:x}_array[] = {{".format(pfx))
     for s in tbl:
         for j in range(2 ** (msl - s.vall)):
-			print("	{} {{{}, {:3d}, {}}},".format(
+            print("{} {{{}, {:3d}, {}}},".format(
                 "\t     " if j else "/* idx {:3d} */".format(s.val + j),
                 s.vall, s.chr % 256,
                 s.esc if s.esc else "NULL"))

doc/sphinx/vtc-syntax.py

@@ -81,4 +81,3 @@ if __name__ == "__main__":
             r = "*"
         print(re.sub(r".", r, tl[section]), end="")
         print("".join(cl[section]))
-

include/Makefile.am

@@ -108,7 +108,7 @@ vcl.h: \
 	    $(top_srcdir)/include/vrt.h \
 	    $(top_srcdir)/doc/sphinx/reference/vcl_var.rst
 	mkdir -p $(top_builddir)/include/tbl
-	@PYTHON@ $(top_srcdir)/lib/libvcc/generate.py \
+	${PYTHON} $(top_srcdir)/lib/libvcc/generate.py \
 	    $(top_srcdir) $(top_builddir)
 
 GEN_H = \
@@ -128,7 +128,7 @@ vcs_version.h:
 	@if test -e $(top_srcdir)/.git || \
 	  ! test -f $(srcdir)/vmod_abi.h || \
 	  ! test -f $(srcdir)/vcs_version.h ; then \
-	    @PYTHON@ $(srcdir)/generate.py \
+	    ${PYTHON} $(srcdir)/generate.py \
 		$(top_srcdir) $(top_builddir) ; \
 	fi
 

include/generate.py

@@ -32,7 +32,6 @@
 from __future__ import print_function
 
 import subprocess
-import collections
 import os
 import sys
 

include/tbl/h2_settings.h

@@ -47,6 +47,7 @@ H2_SETTING(					// rfc7540,l,2097,2103
 	0xffffffff,
 	0
 )
+
 #ifndef H2_SETTINGS_PARAM_ONLY
 H2_SETTING(					// rfc7540,l,2105,2114
 	ENABLE_PUSH,
@@ -58,6 +59,7 @@ H2_SETTING(					// rfc7540,l,2105,2114
 	H2CE_PROTOCOL_ERROR
 )
 #endif
+
 H2_SETTING(					// rfc7540,l,2116,2121
 	MAX_CONCURRENT_STREAMS,
 	max_concurrent_streams,
@@ -67,6 +69,7 @@ H2_SETTING(					// rfc7540,l,2116,2121
 	0xffffffff,
 	0
 )
+
 H2_SETTING(					// rfc7540,l,2139,2148
 	INITIAL_WINDOW_SIZE,
 	initial_window_size,
@@ -76,6 +79,7 @@ H2_SETTING(					// rfc7540,l,2139,2148
 	0x7fffffff,
 	H2CE_FLOW_CONTROL_ERROR
 )
+
 H2_SETTING(					// rfc7540,l,2150,2157
 	MAX_FRAME_SIZE,
 	max_frame_size,
@@ -85,6 +89,7 @@ H2_SETTING(					// rfc7540,l,2150,2157
 	0x00ffffff,
 	H2CE_PROTOCOL_ERROR
 )
+
 H2_SETTING(					// rfc7540,l,2159,2167
 	MAX_HEADER_LIST_SIZE,
 	max_header_list_size,

include/tbl/htc.h

@@ -42,4 +42,5 @@ HTC_STATUS(MORE,	 1,	"more",	"More data required")
 HTC_STATUS(COMPLETE,	 2,	"complete",	"Data complete (no error)")
 HTC_STATUS(IDLE,	 3,	"idle",		"Connection was closed while idle")
 #undef HTC_STATUS
+
 /*lint -restore */

include/tbl/style.py

@@ -7,7 +7,7 @@ from __future__ import print_function
 import glob
 
 def check_file(fn):
-	s = 0
+    print("Check", fn)
     ll = []
     for l in open(fn):
         ll.append(l)
@@ -20,7 +20,9 @@ def check_file(fn):
     assert len(ll) > 5
 
     assert ll.pop(0) == "\n"
-	assert ll.pop(0) == "/*lint -save -e525 -e539 */\n"
+    i = ll.pop(0)
+    assert i == "/*lint -save -e525 -e539 */\n" or \
+           i == "/*lint -save -e525 -e539 -e835 */\n"
     assert ll.pop(0) == "\n"
 
     assert ll.pop(-1) == "/*lint -restore */\n"
@@ -31,7 +33,7 @@ def check_file(fn):
         assert ll[i] != ")\n" or ll[i+1] == "\n" or ll[i+1][0] == "#"
 
     m = {}
-	while len(ll) > 0:
+    while ll:
         i = ll.pop(0)
         if i == "\n":
             continue
@@ -43,7 +45,7 @@ def check_file(fn):
             l = l.split('/*')[0]
             l = l.rstrip()
             if l[-1] != ')':
-				while ll.pop(0) != ')\n':
+                while ll.pop(0).strip() != ')':
                     continue
         elif l[0] == "#":
             j = l[1:].lstrip().split()

include/tbl/vsl_tags.h

@@ -181,16 +181,17 @@ SLTM(Length, 0, "Size of object body",
 	"Logs the size of a fetch object body.\n\n"
 )
 
-/* XXX generate HTC info from tbl include */
-#if 0
-#include <stdio.h>
-int main(void) {
-#define HTC_STATUS(e, n, s, l) \
-	printf("\t\"\\t* %s (%d): %s\\n\"\n", s, n, l);
-#include "include/tbl/htc.h"
-	return (0);
-}
-#endif
+/*
+ * XXX generate HTC info below from tbl include
+ *
+ * #include <stdio.h>
+ * int main(void) {
+ * #define HTC_STATUS(e, n, s, l) \
+ *	printf("\t\"\\t* %s (%d): %s\\n\"\n", s, n, l);
+ * #include "include/tbl/htc.h"
+ *	return (0);
+ * }
+ */
 
 SLTM(FetchError, 0, "Error while fetching object",
 	"Logs the error message of a failed fetch operation.\n\n"

include/tbl/waiters.h

@@ -44,3 +44,5 @@
 
 WAITER(poll)
 #undef WAITER
+
+/*lint -restore */

lib/libvarnishapi/generate.py

@@ -82,13 +82,13 @@ def emit_vxp_fixed_token(fo, tokens):
     emit = dict()
     for i in tokens:
         j = tokens[i]
-		if (j != None):
+        if j is not None:
             recog.append(j)
             emit[j] = i
 
     recog.sort()
     rrecog = copy.copy(recog)
-	rrecog.sort(key = lambda x: -len(x))
+    rrecog.sort(key=lambda x: -len(x))
 
     fo.write("""
 unsigned
@@ -99,23 +99,22 @@ vxp_fixed_token(const char *p, const char **q)
 """)
     last_initial = None
     for i in recog:
-		if (i[0] == last_initial):
+        if i[0] == last_initial:
             continue
         last_initial = i[0]
         fo.write("\tcase '%s':\n" % last_initial)
         for j in rrecog:
-			if (j[0] != last_initial):
+            if j[0] != last_initial:
                 continue
 
             fo.write("\t\tif (")
             k = 1
             l = len(j)
-			while (k < l):
+            while k < l:
                 fo.write("p[%d] == '%s'" % (k, j[k]))
                 fo.write(" &&\n\t\t    ")
                 k += 1
-			fo.write("(isword(p[%d]) ? !isword(p[%d]) : 1)) {\n" %
-				(l - 1, l))
+            fo.write("(isword(p[%d]) ? !isword(p[%d]) : 1)) {\n" % (l - 1, l))
             fo.write("\t\t\t*q = p + %d;\n" % l)
             fo.write("\t\t\treturn (%s);\n" % emit[j])
             fo.write("\t\t}\n")
@@ -132,7 +131,7 @@ def emit_vxp_tnames(fo, tokens):
     l.sort()
     for i in l:
         j = tokens[i]
-		if j == None:
+        if j is None:
             j = i
         if i[0] == "'":
             j = i

lib/libvcc/generate.py

@@ -32,10 +32,6 @@
 
 from __future__ import print_function
 
-import subprocess
-import collections
-import os
-
 #######################################################################
 # These are our tokens
 
@@ -194,9 +190,9 @@ class vardef(object):
 
         # fo.write("\t{ \"%s\", %s,\n" % (nm, self.typ))
         fo.write("\tsym = VCC_MkSym(tl, \"%s\", " % self.nam)
-		if (self.typ == "HEADER"):
+        if self.typ == "HEADER":
             fo.write(" SYM_NONE, %d, %d);\n" % (self.vlo, self.vhi))
-			fo.write("\tAN(sym);\n");
+            fo.write("\tAN(sym);\n")
             fo.write("\tsym->wildcard = vcc_Var_Wildcard;\n")
         else:
             fo.write(" SYM_VAR, %d, %d);\n" % (self.vlo, self.vhi))
@@ -208,7 +204,7 @@ class vardef(object):
             fo.write('\tsym->rname = "HDR_')
             fo.write(self.nam.split(".")[0].upper())
             fo.write('";\n')
-		elif len(self.rd):
+        elif self.rd:
             fo.write('\tsym->rname = "VRT_r_%s(ctx)";\n' % cnam)
             varproto("VCL_" + self.typ + " VRT_r_%s(VRT_CTX)" % cnam)
         fo.write("\tsym->r_methods =\n")
@@ -219,19 +215,19 @@ class vardef(object):
             fo.write('\tsym->lname = "HDR_')
             fo.write(self.nam.split(".")[0].upper())
             fo.write('";\n')
-		elif len(self.wr):
+        elif self.wr:
             fo.write('\tsym->lname = "VRT_l_%s(ctx, ";\n' % cnam)
             s = "void VRT_l_%s(VRT_CTX, " % cnam
             if self.typ != "STRING" and self.typ != "BODY":
                 s += "VCL_" + self.typ + ")"
             else:
                 s += ctyp.c + ", ...)"
-			varproto(s);
+            varproto(s)
         fo.write("\tsym->w_methods =\n")
         restrict(fo, self.wr)
         fo.write(";\n")
 
-		if len(self.uns):
+        if self.uns:
             varproto("void VRT_u_%s(VRT_CTX)" % cnam)
             fo.write('\tsym->uname = "VRT_u_%s(ctx)";\n' % cnam)
         fo.write('\tsym->u_methods =\n')
@@ -239,7 +235,7 @@ class vardef(object):
         fo.write(";\n")
 
 def parse_vcl(x):
-	vlo,vhi = (0,99)
+    vlo, vhi = (0, 99)
     x = x.split()
     if x[0] == "VCL" and x[1] == "<=":
         vhi = int(float(x[2]) * 10)
@@ -249,18 +245,18 @@ def parse_vcl(x):
         print("Unknown variable version spec")
         print("XXX", x, vlo, vhi)
         exit(2)
-	return vlo,vhi
+    return vlo, vhi
 
 def parse_var(ln):
     l1 = ln.pop(0).split("``")
-	assert len(l1) in (1,3)
+    assert len(l1) in (1, 3)
     vn = l1[0].strip()
     if vn[-1] == '*':
         vn = vn[:-1]
     if len(l1) == 3:
-		vlo,vhi = parse_vcl(l1[1])
+        vlo, vhi = parse_vcl(l1[1])
     else:
-		vlo,vhi = 0,99
+        vlo, vhi = 0, 99
     vr = []
     vw = []
     vu = []
@@ -290,7 +286,6 @@ def parse_var(ln):
         vardef(vn, vt, vr, vw, vu, vlo, vhi)
 
 def parse_var_doc(fn):
-	s = 0
     l = []
     for i in open(fn):
         l.append(i.rstrip())
@@ -299,7 +294,7 @@ def parse_var_doc(fn):
         if len(j) != 2 or j[0] != "Type:" or not l[n][0].isspace():
             continue
         m = n
-		while m < len(l) and ( l[m] == "" or l[m][0].isspace() ):
+        while m < len(l) and (l[m] == "" or l[m][0].isspace()):
             m += 1
         parse_var(l[n-2:m-1])
 
@@ -385,17 +380,16 @@ vcl_fixed_token(const char *p, const char **q)
 """)
     last_initial = None
     for i in recog:
-		if (i[0] == last_initial):
+        if i[0] == last_initial:
             continue
         last_initial = i[0]
         fo.write("\tcase '%s':\n" % last_initial)
         need_ret = True
         for j in rrecog:
-			if (j[0] != last_initial):
+            if j[0] != last_initial:
                 continue
             if len(j) == 2:
-				fo.write("\t\tM2('%s', %s);\n" %
-				    (j[1], emit[j]))
+                fo.write("\t\tM2('%s', %s);\n" % (j[1], emit[j]))
             elif len(j) == 1:
                 fo.write("\t\tM1();\n")
                 need_ret = False
@@ -403,7 +397,7 @@ vcl_fixed_token(const char *p, const char **q)
                 fo.write("\t\tif (")
                 k = 1
                 l = len(j)
-				while (k < l):
+                while k < l:
                     fo.write("p[%d] == '%s'" % (k, j[k]))
                     fo.write(" &&")
                     if (k % 3) == 0:
@@ -717,7 +711,7 @@ def restrict(fo, spec):
         fo.write(x)
         w += len(x)
         p = " | "
-	if len(d) == 0:
+    if not d:
         fo.write("0")
 
 #######################################################################

lib/libvcc/vmodtool.py

@@ -41,7 +41,6 @@ import sys
 import re
 import optparse
 import unittest
-import random
 import copy
 import json
 import hashlib

tools/gcov_digest.py

@@ -122,7 +122,7 @@ def run_gcov(prog, subdir):
 
             for ln in x.split("\n"):
                 ln = ln.split()
-				if len(ln) == 0:
+                if not ln:
                     continue
                 if ln[0] == "Creating":
                     gn = ln[1].strip("'")
@@ -148,10 +148,10 @@ def produce_output(fdo):
         lnos.sort()
         pln = -1
         pcn = -1
-		while len(lnos) > 0:
+        while lnos:
             ln, cn = lnos.pop(0)
             lnl = ln
-			while len(lnos) > 0:
+            while lnos:
                 lnn, cnn = lnos[0]
                 if lnl + 1 != lnn or cnn != cn:
                     break
@@ -193,7 +193,7 @@ if __name__ == "__main__":
             exclude.append(v)
         else:
             assert False
-	if len(args) == 0:
+    if not args:
         args = ["."]
     for dn in args:
         run_gcov(gcovprog, dn)

tools/include_wash.py

@@ -8,7 +8,7 @@ def check(fn):
     l = []
     for i in open(fn):
         i = i.strip()
-		if len(i) == 0:
+        if not i:
             continue
         if i[0] != "#":
             continue
@@ -20,7 +20,7 @@ def check(fn):
             l.append(i.split('"')[1])
     if "vrt.h" in l:
         vrt = l.index("vrt.h")
-		if not "vdef.h" in l:
+        if "vdef.h" not in l:
             print(fn, "vdef.h not included with vrt.h")
         vdef = l.index("vdef.h")
         if vdef > vrt:
@@ -31,11 +31,9 @@ def check(fn):
 
     for i in ("cache/cache.h", "cache.h"):
         if i in l:
-			for i in (
-			    "stddef.h", "stdint.h", "vrt.h",
+            for i in ("stddef.h", "stdint.h", "vrt.h",
                       "math.h", "pthread.h", "stdarg.h", "sys/types.h",
-			    "vdef.h", "miniobj.h", "vas.h", "vqueue.h",
-			):
+                      "vdef.h", "miniobj.h", "vas.h", "vqueue.h"):
                 if i in l:
                     print(fn, i + " included with cache.h")