feat: initial HSAP platform

Huaxu Sentinel Active Safety Platform with embedded algorithm code,
Docker Compose setup, and vendored dataset scaffolds for clone-and-run.

Co-authored-by: Cursor <cursoragent@cursor.com>

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/__init__.py

@@ -0,0 +1,7 @@
+from . import datasets
+from . import losses
+from . import transforms
+from . import models
+from . import lr_schedulers
+from . import optimizers
+# from . import ops

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/args.py

@@ -0,0 +1,123 @@
+import os
+from importlib.machinery import SourceFileLoader
+
+from configs.statics import DEPRECATION_MAP, SHORTCUTS
+try:
+    from .common import warnings
+except ImportError:
+    import warnings
+
+
+def update_nested(d, keys, value):
+    # Update nested dict with keys as list
+    if not isinstance(d, dict):
+        raise ValueError
+    if len(keys) == 1:
+        d[keys[0]] = value
+    else:
+        if keys[0] in d.keys():
+            d[keys[0]] = update_nested(d[keys[0]], keys[1:], value)
+        else:
+            d[keys[0]] = update_nested({}, keys[1:], value)
+
+    return d
+
+
+def cmd_dict(x):
+    # A data type to hack a dict-style argparse input,
+    # every key should be in k1.k2.kn format to the last non-dict value,
+    # values can't include tuples,
+    # other more complex settings should refer to config files instead.
+    # x: x1=y1 x2=y2 x3=y3 etc.
+    options = x.split()
+    res = {}
+    for o in options:
+        kv = o.split('=', maxsplit=1)
+        try:
+            v = eval(kv[1])
+        except:
+            v = kv[1]
+        res[kv[0]] = v
+
+    return res
+
+
+def add_shortcuts(parser):
+    # TODO: duplicates
+    for k, v in SHORTCUTS.items():
+        parser.add_argument('--' + k.replace('_', '-'), type=v['type'],
+                            help='{}. Shortcut for {}'.format(v['help'], str(v['keys'])))
+
+
+def read_config(config_path):
+    # Read a mmlab-style python config file and parse to a single dict
+    module_name = os.path.split(config_path)[1]
+    assert module_name[-3:] == '.py'
+    module_name = module_name[:-3]
+    module = SourceFileLoader(module_name, config_path).load_module()
+    res = {k: v for k, v in module.__dict__.items() if ((not k.startswith('__')) and (not callable(v)))}
+
+    return res
+
+
+def map_states(args, states):
+    # Map --train --test etc. to args.state
+    state = args.state
+    for k, v in vars(args).items():
+        if k in states and v:  # Map states
+            state = states.index(k)
+
+    return state
+
+
+def parse_arg_cfg(args, cfg, deprecation_map=None):
+    if deprecation_map is None:
+        deprecation_map = DEPRECATION_MAP
+
+    # Simply reject deprecations
+    dict_args = vars(args)
+    if deprecation_map is not None:
+        for deprecated, v in deprecation_map.items():
+            if dict_args.get(deprecated) is not None:
+                if v['valid'] is None:  # Not used anymore
+                    warnings.warn('Deprecated arg {}={} will not be used. '.format(deprecated, cfg[deprecated])
+                                  + v['message'])
+                else:  # Use the deprecated arg in absence of new arg
+                    warnings.warn('Arg {} is deprecated, please use {} instead. '.format(deprecated, v['valid'])
+                                  + v['message'])
+                    if dict_args.get(v['valid']) is None:
+                        dict_args[v['valid']] = dict_args[deprecated]
+
+    # Set shortcuts
+    overrides = dict_args['cfg_options']
+    if overrides is None:
+        overrides = {}
+    for k, v in dict_args.items():
+        if k in SHORTCUTS.keys():
+            for tk in SHORTCUTS[k]['keys']:
+                v_cfg_options = overrides.get(tk)
+                if v_cfg_options is not None:
+                    if v is not None and v != v_cfg_options:
+                        raise ValueError('Conflict between arg {}={} in --cfg-option and shortcut arg {}={}'.format(
+                            tk, v_cfg_options, k, v
+                        ))
+                else:
+                    overrides[tk] = v
+
+    # Override cfg by args
+    for k, v in overrides.items():
+        if v is not None:
+            if type(v) == bool:
+                warnings.warn('Override Bool arg {} is insecure, by default, it will be overridden by False!'.format(
+                    k
+                ))
+            # k = 'k1.k2.kn'
+            key_path = k.split('.')
+            try:
+                cfg = update_nested(cfg, key_path, v)
+            except RuntimeError:
+                raise RuntimeError('Structural conflict in config key path {}!'.format(key_path))
+
+    # Add retain args
+
+    return args, cfg

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/common.py

@@ -0,0 +1,69 @@
+import torch
+from collections import OrderedDict
+
+from .ddp_utils import save_on_master
+
+
+def get_warnings():
+    # Get rid of the extra line of code printing
+    # https://stackoverflow.com/a/26433913/15449902
+    import warnings
+
+    def warning_on_one_line(message, category, filename, lineno, file=None, line=None):
+        return '%s:%s: %s: %s\n' % (filename, lineno, category.__name__, message)
+
+    warnings.formatwarning = warning_on_one_line
+
+    return warnings
+
+
+warnings = get_warnings()
+
+
+# Save model checkpoints (supports amp)
+def save_checkpoint(net, optimizer, lr_scheduler, filename='temp.pt'):
+    checkpoint = {
+        'model': net.state_dict(),
+        'optimizer': optimizer.state_dict() if optimizer is not None else None,
+        'lr_scheduler': lr_scheduler.state_dict() if lr_scheduler is not None else None
+    }
+    save_on_master(checkpoint, filename)
+
+
+# Load model checkpoints (supports amp)
+def load_checkpoint(net, optimizer, lr_scheduler, filename, strict=True):
+    try:
+        checkpoint = torch.load(filename, map_location='cpu')
+    except:
+        warnings.warn('Model not saved as on cpu, could be a legacy trained weight, trying loading on saved device...')
+        checkpoint = torch.load(filename)
+        print('Loaded on saved device.')
+
+    # To keep BC while having a acceptable variable name for lane detection
+    checkpoint['model'] = OrderedDict((k.replace('aux_head', 'lane_classifier') if 'aux_head' in k else k, v)
+                                      for k, v in checkpoint['model'].items())
+    # state_dict = checkpoint['model']
+    # self_state_dict = net.state_dict()
+    # self_keys = list(self_state_dict.keys())
+    # for i, (_, v) in enumerate(state_dict.items()):
+    #     if i > len(self_keys) - 1:
+    #         break
+    #     self_state_dict[self_keys[i]] = v
+    #
+    # # for k, v in state_dict.items():
+    # #     print(k)
+    # # quit(0)
+    net.load_state_dict(checkpoint['model'], strict=strict)
+
+    if optimizer is not None:
+        try:  # Shouldn't be necessary, but just in case
+            optimizer.load_state_dict(checkpoint['optimizer'])
+        except RuntimeError:
+            warnings.warn('Incorrect optimizer state dict, maybe you are using old code with aux_head?')
+            pass
+    if lr_scheduler is not None:
+        try:  # Shouldn't be necessary, but just in case
+            lr_scheduler.load_state_dict(checkpoint['lr_scheduler'])
+        except RuntimeError:
+            warnings.warn('Incorrect lr scheduler state dict, maybe you are using old code with aux_head?')
+            pass

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/csrc/__init__.py

@@ -0,0 +1 @@
+from . import apis

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/csrc/apis.py

@@ -0,0 +1,17 @@
+import os
+from torch.utils.cpp_extension import load
+
+csrc_path = 'utils/csrc'
+
+line_nms_ = load(name='line_nms',
+                 sources=[os.path.join(csrc_path, 'line_nms', 'line_nms.cpp'),
+                          os.path.join(csrc_path, 'line_nms', 'line_nms_kernel.cu')],
+                 verbose=False)
+
+
+# Wrap it to be like a normal Python func
+# TODO: cpu version
+def line_nms(boxes, scores, overlap, top_k):
+    # Notes: removed the extra 5 (start, end, len, valid (2)) in original LaneATT
+
+    return line_nms_.forward(boxes.contiguous(), scores.contiguous(), overlap, top_k)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/csrc/line_nms/ORIGINAL_LICENSE

@@ -0,0 +1,59 @@
+Copyright (c) 2018, Grégoire Payen de La Garanderie, Durham University
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+************************************************************************
+
+THIRD-PARTY SOFTWARE NOTICES AND INFORMATION
+
+This project incorporates material from the project(s)
+listed below (collectively, "Third Party Code"). This Third Party Code is
+licensed to you under their original license terms set forth below.
+
+1. Faster R-CNN, (https://github.com/rbgirshick/py-faster-rcnn)
+
+The MIT License (MIT)
+
+Copyright (c) 2015 Microsoft Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/csrc/line_nms/line_nms.cpp

@@ -0,0 +1,32 @@
+#include <torch/extension.h>
+#include <torch/types.h>
+#include <iostream>
+
+std::vector<at::Tensor> nms_cuda_forward(
+        at::Tensor boxes,
+        at::Tensor idx,
+        float nms_overlap_thresh,
+        unsigned long top_k);
+
+#define CHECK_CUDA(x) AT_ASSERTM(x.type().is_cuda(), #x " must be a CUDA tensor")
+#define CHECK_CONTIGUOUS(x) AT_ASSERTM(x.is_contiguous(), #x " must be contiguous")
+#define CHECK_INPUT(x) CHECK_CUDA(x); CHECK_CONTIGUOUS(x)
+
+std::vector<at::Tensor> nms_forward(
+        at::Tensor boxes,
+        at::Tensor scores,
+        float thresh,
+        unsigned long top_k) {
+
+
+    auto idx = std::get<1>(scores.sort(0,true));
+
+    CHECK_INPUT(boxes);
+    CHECK_INPUT(idx);
+
+    return nms_cuda_forward(boxes, idx, thresh, top_k);
+}
+
+PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
+    m.def("forward", &nms_forward, "NMS forward");
+}

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/csrc/line_nms/line_nms_kernel.cu

@@ -0,0 +1,192 @@
+#include <torch/extension.h>
+#include <ATen/ATen.h>
+
+#include <cuda.h>
+#include <cuda_runtime.h>
+#include <vector>
+#include <iostream>
+
+// Hard-coded maximum. Increase if needed.
+#define MAX_COL_BLOCKS 1000
+#define STRIDE 4
+#define N_OFFSETS 72 // if you use more than 72 offsets you will have to adjust this value
+#define N_STRIPS (N_OFFSETS - 1)
+#define PROP_SIZE (N_OFFSETS + 2)  // start, len, 72 offsets
+#define DATASET_OFFSET 0
+
+#define DIVUP(m,n) (((m)+(n)-1) / (n))
+int64_t const threadsPerBlock = sizeof(unsigned long long) * 8;
+
+// The functions below originates from Fast R-CNN
+// See https://github.com/rbgirshick/py-faster-rcnn
+// Copyright (c) 2015 Microsoft
+// Licensed under The MIT License
+// Written by Shaoqing Ren
+
+template <typename scalar_t>
+// __device__ inline scalar_t devIoU(scalar_t const * const a, scalar_t const * const b) {
+__device__ inline bool devIoU(scalar_t const * const a, scalar_t const * const b, const float threshold) {
+  const int start_a = (int) (a[0] * N_STRIPS - DATASET_OFFSET + 0.5); // 0.5 rounding trick
+  const int start_b = (int) (b[0] * N_STRIPS - DATASET_OFFSET + 0.5);
+  const int start = max(start_a, start_b);
+  const int end_a = start_a + a[1] - 1 + 0.5 - ((a[4] - 1) < 0); //  - (x<0) trick to adjust for negative numbers (in case length is 0)
+  const int end_b = start_b + b[1] - 1 + 0.5 - ((b[4] - 1) < 0);
+  const int end = min(min(end_a, end_b), N_OFFSETS - 1);
+  // if (end < start) return 1e9;
+  if (end < start) return false;
+  scalar_t dist = 0;
+  for(unsigned char i = 2 + start; i <= 2 + end; ++i) {
+    if (a[i] < b[i]) {
+      dist += b[i] - a[i];
+    } else {
+      dist += a[i] - b[i];
+    }
+  }
+  // return (dist / (end - start + 1)) < threshold;
+  return dist < (threshold * (end - start + 1));
+  // return dist / (end - start + 1);
+}
+
+template <typename scalar_t>
+__global__ void nms_kernel(const int64_t n_boxes, const scalar_t nms_overlap_thresh,
+                           const scalar_t *dev_boxes, const int64_t *idx, int64_t *dev_mask) {
+  const int64_t row_start = blockIdx.y;
+  const int64_t col_start = blockIdx.x;
+
+  if (row_start > col_start) return;
+
+  const int row_size =
+        min(n_boxes - row_start * threadsPerBlock, threadsPerBlock);
+  const int col_size =
+        min(n_boxes - col_start * threadsPerBlock, threadsPerBlock);
+
+  __shared__ scalar_t block_boxes[threadsPerBlock * PROP_SIZE];
+  if (threadIdx.x < col_size) {
+    for (int i = 0; i <  PROP_SIZE; ++i) {
+      block_boxes[threadIdx.x * PROP_SIZE + i] = dev_boxes[idx[(threadsPerBlock * col_start + threadIdx.x)] * PROP_SIZE + i];
+    }
+  //   block_boxes[threadIdx.x * 4 + 0] =
+  //       dev_boxes[idx[(threadsPerBlock * col_start + threadIdx.x)] * 4 + 0];
+  //   block_boxes[threadIdx.x * 4 + 1] =
+  //       dev_boxes[idx[(threadsPerBlock * col_start + threadIdx.x)] * 4 + 1];
+  //   block_boxes[threadIdx.x * 4 + 2] =
+  //       dev_boxes[idx[(threadsPerBlock * col_start + threadIdx.x)] * 4 + 2];
+  //   block_boxes[threadIdx.x * 4 + 3] =
+  //       dev_boxes[idx[(threadsPerBlock * col_start + threadIdx.x)] * 4 + 3];
+  }
+  __syncthreads();
+
+  if (threadIdx.x < row_size) {
+    const int cur_box_idx = threadsPerBlock * row_start + threadIdx.x;
+    const scalar_t *cur_box = dev_boxes + idx[cur_box_idx] * PROP_SIZE;
+    int i = 0;
+    unsigned long long t = 0;
+    int start = 0;
+    if (row_start == col_start) {
+      start = threadIdx.x + 1;
+    }
+    for (i = start; i < col_size; i++) {
+      if (devIoU(cur_box, block_boxes + i * PROP_SIZE, nms_overlap_thresh)) {
+        t |= 1ULL << i;
+      }
+    }
+    const int col_blocks = DIVUP(n_boxes, threadsPerBlock);
+    dev_mask[cur_box_idx * col_blocks + col_start] = t;
+  }
+}
+
+
+__global__ void nms_collect(const int64_t boxes_num, const int64_t col_blocks, int64_t top_k, const int64_t *idx, const int64_t *mask, int64_t *keep, int64_t *parent_object_index, int64_t *num_to_keep) {
+  int64_t remv[MAX_COL_BLOCKS];
+  int64_t num_to_keep_ = 0;
+
+  for (int i = 0; i < col_blocks; i++) {
+      remv[i] = 0;
+  }
+
+  for (int i = 0; i < boxes_num; ++i) {
+      parent_object_index[i] = 0;
+  }
+
+  for (int i = 0; i < boxes_num; i++) {
+    int nblock = i / threadsPerBlock;
+    int inblock = i % threadsPerBlock;
+
+
+    if (!(remv[nblock] & (1ULL << inblock))) {
+      int64_t idxi = idx[i];
+      keep[num_to_keep_] = idxi;
+      const int64_t *p = &mask[0] + i * col_blocks;
+      for (int j = nblock; j < col_blocks; j++) {
+        remv[j] |= p[j];
+      }
+      for (int j = i; j < boxes_num; j++) {
+        int nblockj = j / threadsPerBlock;
+        int inblockj = j % threadsPerBlock;
+        if (p[nblockj] & (1ULL << inblockj))
+            parent_object_index[idx[j]] = num_to_keep_+1;
+      }
+      parent_object_index[idx[i]] = num_to_keep_+1;
+
+      num_to_keep_++;
+
+      if (num_to_keep_==top_k)
+          break;
+    }
+  }
+
+  // Initialize the rest of the keep array to avoid uninitialized values.
+  for (int i = num_to_keep_; i < boxes_num; ++i)
+      keep[i] = 0;
+
+  *num_to_keep = min(top_k,num_to_keep_);
+}
+
+#define CHECK_CONTIGUOUS(x) AT_ASSERTM(x.is_contiguous(), #x " must be contiguous")
+
+std::vector<at::Tensor> nms_cuda_forward(
+        at::Tensor boxes,
+        at::Tensor idx,
+        float nms_overlap_thresh,
+        unsigned long top_k) {
+
+  const auto boxes_num = boxes.size(0);
+  TORCH_CHECK(boxes.size(1) == PROP_SIZE, "Wrong number of offsets. Please adjust `PROP_SIZE`");
+
+  const int col_blocks = DIVUP(boxes_num, threadsPerBlock);
+
+  AT_ASSERTM (col_blocks < MAX_COL_BLOCKS, "The number of column blocks must be less than MAX_COL_BLOCKS. Increase the MAX_COL_BLOCKS constant if needed.");
+
+  auto longOptions = torch::TensorOptions().device(torch::kCUDA).dtype(torch::kLong);
+  auto mask = at::empty({boxes_num * col_blocks}, longOptions);
+
+  dim3 blocks(DIVUP(boxes_num, threadsPerBlock),
+              DIVUP(boxes_num, threadsPerBlock));
+  dim3 threads(threadsPerBlock);
+
+  CHECK_CONTIGUOUS(boxes);
+  CHECK_CONTIGUOUS(idx);
+  CHECK_CONTIGUOUS(mask);
+
+  AT_DISPATCH_FLOATING_TYPES(boxes.type(), "nms_cuda_forward", ([&] {
+    nms_kernel<<<blocks, threads>>>(boxes_num,
+                                    (scalar_t)nms_overlap_thresh,
+                                    boxes.data<scalar_t>(),
+                                    idx.data<int64_t>(),
+                                    mask.data<int64_t>());
+  }));
+
+  auto keep = at::empty({boxes_num}, longOptions);
+  auto parent_object_index = at::empty({boxes_num}, longOptions);
+  auto num_to_keep = at::empty({}, longOptions);
+
+  nms_collect<<<1, 1>>>(boxes_num, col_blocks, top_k,
+                        idx.data<int64_t>(),
+                        mask.data<int64_t>(),
+                        keep.data<int64_t>(),
+                        parent_object_index.data<int64_t>(),
+                        num_to_keep.data<int64_t>());
+
+
+  return {keep,num_to_keep,parent_object_index};
+}

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/curve_utils.py

@@ -0,0 +1,213 @@
+import torch
+import numpy as np
+from scipy.interpolate import splprep, splev
+from scipy.special import comb as n_over_k
+
+
+def upcast(t):
+    # Protects from numerical overflows in multiplications by upcasting to the equivalent higher type
+    # https://github.com/pytorch/vision/pull/3383
+    if t.is_floating_point():
+        return t if t.dtype in (torch.float32, torch.float64) else t.float()
+    else:
+        return t if t.dtype in (torch.int32, torch.int64) else t.int()
+
+
+class Polynomial(object):
+    # Define Polynomials for curve fitting 
+    def __init__(self, order):
+        self.order = order
+
+    def poly_fit(self, x_list, y_list, interpolate=False):
+        self.coeff = np.polyfit(y_list, x_list, self.order)
+
+    def compute_x_based_y(self, y, image_size):
+        out = 0
+        for i in range(self.order + 1):
+            out += (y ** (self.order - i)) * self.coeff[i]
+        if image_size is not None:
+            out = out * image_size[-1]
+
+        return out
+
+    def print_coeff(self):
+        print(self.coeff)
+
+    def get_sample_point(self, y_list, image_size):
+        coord_list = []
+        for y in y_list:
+            x = self.compute_x_based_y(y, None)
+            coord_list.append([round(x, 3), y])
+        coord_list = np.array(coord_list)
+        if image_size is not None:
+            coord_list[:, 0] = coord_list[:, 0] * image_size[-1]
+            coord_list[:, -1] = coord_list[:, -1] * image_size[0]
+
+        return coord_list
+
+
+class BezierCurve(object):
+    # Define Bezier curves for curve fitting
+    def __init__(self, order, num_sample_points=50):
+        self.num_point = order + 1
+        self.control_points = []
+        self.bezier_coeff = self.get_bezier_coefficient()
+        self.num_sample_points = num_sample_points
+        self.c_matrix = self.get_bernstein_matrix()
+
+    def get_bezier_coefficient(self):
+        Mtk = lambda n, t, k: t ** k * (1 - t) ** (n - k) * n_over_k(n, k)
+        BezierCoeff = lambda ts: [[Mtk(self.num_point - 1, t, k) for k in range(self.num_point)] for t in ts]
+
+        return BezierCoeff
+
+    def interpolate_lane(self, x, y, n=50):
+        # Spline interpolation of a lane. Used on the predictions
+        assert len(x) == len(y)
+
+        tck, _ = splprep([x, y], s=0, t=n, k=min(3, len(x) - 1))
+
+        u = np.linspace(0., 1., n)
+        return np.array(splev(u, tck)).T
+
+    def get_control_points(self, x, y, interpolate=False):
+        if interpolate:
+            points = self.interpolate_lane(x, y)
+            x = np.array([x for x, _ in points])
+            y = np.array([y for _, y in points])
+
+        middle_points = self.get_middle_control_points(x, y)
+        for idx in range(0, len(middle_points) - 1, 2):
+            self.control_points.append([middle_points[idx], middle_points[idx + 1]])
+
+    def get_bernstein_matrix(self):
+        tokens = np.linspace(0, 1, self.num_sample_points)
+        c_matrix = self.bezier_coeff(tokens)
+        return np.array(c_matrix)
+
+    def save_control_points(self):
+        return self.control_points
+
+    def assign_control_points(self, control_points):
+        self.control_points = control_points
+
+    def quick_sample_point(self, image_size=None):
+        control_points_matrix = np.array(self.control_points)
+        sample_points = self.c_matrix.dot(control_points_matrix)
+        if image_size is not None:
+            sample_points[:, 0] = sample_points[:, 0] * image_size[-1]
+            sample_points[:, -1] = sample_points[:, -1] * image_size[0]
+        return sample_points
+
+    def get_sample_point(self, n=50, image_size=None):
+        '''
+            :param n: the number of sampled points
+            :return: a list of sampled points
+        '''
+        t = np.linspace(0, 1, n)
+        coeff_matrix = np.array(self.bezier_coeff(t))
+        control_points_matrix = np.array(self.control_points)
+        sample_points = coeff_matrix.dot(control_points_matrix)
+        if image_size is not None:
+            sample_points[:, 0] = sample_points[:, 0] * image_size[-1]
+            sample_points[:, -1] = sample_points[:, -1] * image_size[0]
+
+        return sample_points
+
+    def get_middle_control_points(self, x, y):
+        dy = y[1:] - y[:-1]
+        dx = x[1:] - x[:-1]
+        dt = (dx ** 2 + dy ** 2) ** 0.5
+        t = dt / dt.sum()
+        t = np.hstack(([0], t))
+        t = t.cumsum()
+        data = np.column_stack((x, y))
+        Pseudoinverse = np.linalg.pinv(self.bezier_coeff(t))  # (9,4) -> (4,9)
+        control_points = Pseudoinverse.dot(data)  # (4,9)*(9,2) -> (4,2)
+        medi_ctp = control_points[:, :].flatten().tolist()
+
+        return medi_ctp
+
+
+class BezierSampler(torch.nn.Module):
+    # Fast Batch Bezier sampler
+    def __init__(self, order, num_sample_points, proj_coefficient=0):
+        super().__init__()
+        self.proj_coefficient = proj_coefficient
+        self.num_control_points = order + 1
+        self.num_sample_points = num_sample_points
+        self.control_points = []
+        self.bezier_coeff = self.get_bezier_coefficient()
+        self.bernstein_matrix = self.get_bernstein_matrix()
+
+    def get_bezier_coefficient(self):
+        Mtk = lambda n, t, k: t ** k * (1 - t) ** (n - k) * n_over_k(n, k)
+        BezierCoeff = lambda ts: [[Mtk(self.num_control_points - 1, t, k) for k in range(self.num_control_points)] for t
+                                  in ts]
+        return BezierCoeff
+
+    def get_bernstein_matrix(self):
+        t = torch.linspace(0, 1, self.num_sample_points)
+        if self.proj_coefficient != 0:
+            # tokens = tokens + (1 - tokens) * tokens ** self.proj_coefficient
+            t[t > 0.5] = t[t > 0.5] + (1 - t[t > 0.5]) * t[t > 0.5] ** self.proj_coefficient
+            t[t < 0.5] = 1 - (1 - t[t < 0.5] + t[t < 0.5] * (1 - t[t < 0.5]) ** self.proj_coefficient)
+        c_matrix = torch.tensor(self.bezier_coeff(t))
+        return c_matrix
+
+    def get_sample_points(self, control_points_matrix):
+        if control_points_matrix.numel() == 0:
+            return control_points_matrix  # Looks better than a torch.Tensor
+        if self.bernstein_matrix.device != control_points_matrix.device:
+            self.bernstein_matrix = self.bernstein_matrix.to(control_points_matrix.device)
+
+        return upcast(self.bernstein_matrix).matmul(upcast(control_points_matrix))
+
+
+@torch.no_grad()
+def get_valid_points(points):
+    # ... x 2
+    if points.numel() == 0:
+        return torch.tensor([1], dtype=torch.bool, device=points.device)
+    return (points[..., 0] > 0) * (points[..., 0] < 1) * (points[..., 1] > 0) * (points[..., 1] < 1)
+
+
+@torch.no_grad()
+def cubic_bezier_curve_segment(control_points, sample_points):
+    # Cut a batch of cubic bezier curves to its in-image segments (assume at least 2 valid sample points per curve).
+    # Based on De Casteljau's algorithm, formula for cubic bezier curve is derived by:
+    # https://stackoverflow.com/a/11704152/15449902
+    # control_points: B x 4 x 2
+    # sample_points: B x N x 2
+    if control_points.numel() == 0 or sample_points.numel() == 0:
+        return control_points
+    B, N = sample_points.shape[:-1]
+    valid_points = get_valid_points(sample_points)  # B x N, bool
+    t = torch.linspace(0.0, 1.0, steps=N, dtype=sample_points.dtype, device=sample_points.device)
+
+    # First & Last valid index (B)
+    # Get unique values for deterministic behaviour on cuda:
+    # https://pytorch.org/docs/1.6.0/generated/torch.max.html?highlight=max#torch.max
+    t0 = t[(valid_points + torch.arange(N, device=valid_points.device).flip([0]) * valid_points).max(dim=-1).indices]
+    t1 = t[(valid_points + torch.arange(N, device=valid_points.device) * valid_points).max(dim=-1).indices]
+
+    # Generate transform matrix (old control points -> new control points = linear transform)
+    u0 = 1 - t0  # B
+    u1 = 1 - t1  # B
+    transform_matrix_c = [torch.stack([u0 ** (3 - i) * u1 ** i for i in range(4)], dim=-1),
+                          torch.stack([3 * t0 * u0 ** 2,
+                                       2 * t0 * u0 * u1 + u0 ** 2 * t1,
+                                       t0 * u1 ** 2 + 2 * u0 * u1 * t1,
+                                       3 * t1 * u1 ** 2], dim=-1),
+                          torch.stack([3 * t0 ** 2 * u0,
+                                       t0 ** 2 * u1 + 2 * t0 * t1 * u0,
+                                       2 * t0 * t1 * u1 + t1 ** 2 * u0,
+                                       3 * t1 ** 2 * u1], dim=-1),
+                          torch.stack([t0 ** (3 - i) * t1 ** i for i in range(4)], dim=-1)]
+    transform_matrix = torch.stack(transform_matrix_c, dim=-2).transpose(-2, -1)  # B x 4 x 4, f**k this!
+    transform_matrix = transform_matrix.unsqueeze(1).expand(B, 2, 4, 4)
+
+    # Matrix multiplication
+    res = transform_matrix.matmul(control_points.permute(0, 2, 1).unsqueeze(-1))  # B x 2 x 4 x 1
+
+    return res.squeeze(-1).permute(0, 2, 1)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/custom_op_flop_counters.py

@@ -0,0 +1,45 @@
+import torch
+
+
+def hook_DCN_v2_Ref(m, x, y):
+    # Only modulated_deform_conv2d() is ignored in feature flip fusion,
+    # same as default DCNv2.
+    # conv Flops = 2 x k^2 x Cin x Cout x W x H
+    # dcn Flops = + W x H x k^2 x 13 + W x H x k^2 x 9 x Cin (bilinear interpolate, 9 -> 6 for MACs)
+    # or just: ->0 + W x H x k^2 x 9 x Cin (3 weighted plus, 9 -> 6 for MACs)
+    # ->0 coordinates: W x H x k^2 x 13
+    # DCNv2: ++ W x H x k^2 x Cin
+    # ~ (additional flops on bias is not considered):
+    x = x[0]
+    C_in, fH, fW = x.shape[1:]
+    C_out = y.shape[1]
+
+    # 1. Offset conv
+    bias_ops = 1 if m.conv_offset.bias is not None else 0
+
+    # N x Cout x H x W x  (Cin x Kw x Kh + bias)
+    m.total_ops += counter_conv(
+        bias_ops,
+        torch.zeros(m.conv_offset.weight.size()[2:]).numel(),
+        int(m.conv_offset.weight.size()[0] * fH * fW),
+        m.conv_offset.in_channels,
+        m.conv_offset.groups,
+    )
+
+    # 2. sigmoid on DCNv2 mask (doubly count as in thop)
+    m.total_ops += torch.DoubleTensor([19 * m.conv_offset.weight.size()[1] * fH * fW // 3])
+
+    # 3. DCNv2
+    m.total_ops += torch.DoubleTensor([count_one_dcn_v2(fH, fW, C_in, C_out)])
+
+
+def counter_conv(bias, kernel_size, output_size, in_channel, group):
+    """inputs are all numbers!"""
+    return torch.DoubleTensor([output_size * (in_channel / group * kernel_size + bias)])
+
+
+def count_one_dcn_v2(fW, fH, C_in, C_out):
+    standard_conv_macs = fW * fH * (3 * 3) * C_in * C_out
+    ignored_dcnv2_macs = 7 * fW * fH * (3 * 3) * C_in + fW * fH * (3 * 3) * 13
+
+    return standard_conv_macs + ignored_dcnv2_macs

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/datasets/__init__.py

@@ -0,0 +1,13 @@
+from .segmentation import PASCAL_VOC_Segmentation, CityscapesSegmentation, SYNTHIA_Segmentation, GTAV_Segmentation
+from .lane_as_segmentation import TuSimpleAsSegmentation, CULaneAsSegmentation, LLAMAS_AsSegmentation
+from .lane_as_bezier import TuSimpleAsBezier, CULaneAsBezier, LLAMAS_AsBezier, Curvelanes_AsBezier
+from .tusimple import TuSimple
+from .tusimple_vis import TuSimpleVis
+from .culane import CULane
+from .culane_vis import CULaneVis
+from .llamas import LLAMAS
+from .llamas_vis import LLAMAS_Vis
+from .image_folder import ImageFolderDataset
+from .video import VideoLoader
+from .utils import dict_collate_fn
+from .builder import DATASETS

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/datasets/bdd100k.py

@@ -0,0 +1,22 @@
+import torchvision
+import os
+import numpy as np
+from PIL import Image
+
+
+# BDD100K direct loading (work with the segmentation style lists)
+class CULane(torchvision.datasets.VisionDataset):
+    def __init__(self, root, image_set, transforms=None, transform=None, target_transform=None,
+                 ppl=0, gap=0, start=0):
+        super().__init__(root, transforms, transform, target_transform)
+        pass
+
+    def __getitem__(self, index):
+        # Return x (input image) & y (L lane with N coordinates (x, y) as np.array (L x N x 2))
+        # Empty coordinates are marked by (-2, -2)
+        # If just testing,
+        # y is the filename to store prediction
+        pass
+
+    def __len__(self):
+        pass

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/datasets/builder.py

@@ -0,0 +1,3 @@
+from ..registry import SimpleRegistry
+
+DATASETS = SimpleRegistry()

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/datasets/culane.py

@@ -0,0 +1,74 @@
+import os
+import pickle
+import numpy as np
+from tqdm import tqdm
+
+from .utils import LaneKeypointDataset
+from .builder import DATASETS
+
+
+# CULane direct loading (work with the segmentation style lists)
+@DATASETS.register()
+class CULane(LaneKeypointDataset):
+    colors = [
+        [0, 0, 0],  # background
+        [0, 255, 0], [0, 0, 255], [255, 0, 0], [255, 255, 0],
+        [0, 0, 0]  # ignore
+    ]
+
+    def __init__(self, root, image_set, transforms=None, transform=None, target_transform=None,
+                 ppl=31, gap=10, start=290, padding_mask=False, is_process=True):
+        super().__init__(root, transforms, transform, target_transform, ppl, gap, start, padding_mask, image_set,
+                         is_process)
+
+        self._check()
+
+        # Data list
+        with open(os.path.join(root, 'lists', image_set + '.txt'), "r") as f:
+            contents = [x.strip() for x in f.readlines()]
+
+        # Load filenames
+        if image_set == 'test' or image_set == 'val':  # Test
+            self.images = [os.path.join(root, x + '.jpg') for x in contents]
+            self.targets = [os.path.join('./output', x + '.lines.txt') for x in contents]
+        else:  # Train
+            self.images = [os.path.join(root, x[:x.find(' ')] + '.jpg') for x in contents]
+            self.targets = []
+            print('Loading targets into memory...')
+            processed_file = os.path.join(root, 'train_processed_targets')
+            if os.path.exists(processed_file):
+                with open(processed_file, 'rb') as f:
+                    self.targets = pickle.load(f)
+            else:
+                print('Pre-processing will only be performed for 1 time, please wait ~10 minutes.')
+                for x in tqdm(contents):
+                    with open(os.path.join(root, x[:x.find(' ')] + '.lines.txt'), 'r') as f:
+                        self.targets.append(self._load_target(f.readlines()))
+                with open(processed_file, 'wb') as f:
+                    pickle.dump(self.targets, f)
+            print('Loading complete.')
+
+        assert len(self.targets) == len(self.images)
+
+    def _load_target(self, lines):
+        # Read file content to lists (file content could be empty or variable number of lanes)
+        target = np.array([[[-2.0, self.start + i * self.gap] for i in range(self.ppl)]
+                          for _ in range(len(lines))], dtype=np.float32)
+        for i in range(len(lines)):  # lines=[] will end this immediately
+            temp = [float(k) for k in lines[i].strip().split(' ')]
+            for j in range(int(len(temp) / 2)):
+                x = temp[2 * j]
+                y = temp[2 * j + 1]
+                target[i][target[i][:, 1] == y] = [x, y]
+
+        return target
+
+    @staticmethod
+    def load_target_xy(lines):
+        # A direct loading of JSON file to a list of N x 2 numpy arrays
+        target = []
+        for line in lines:
+            temp = [float(x) for x in line.strip().split(' ')]
+            target.append(np.array(temp).reshape(-1, 2))
+
+        return target

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/datasets/culane_vis.py

@@ -0,0 +1,47 @@
+import os
+
+from .image_folder_lane_base import ImageFolderLaneBase
+from .builder import DATASETS
+
+
+# Visualization version of CULane
+@DATASETS.register()
+class CULaneVis(ImageFolderLaneBase):
+    def __init__(self, root_dataset, root_output, root_keypoint, image_set, transforms=None,
+                 keypoint_process_fn=None, use_gt=True):
+        super().__init__(root_dataset, root_output, transforms, keypoint_process_fn)
+        self.image_set = image_set
+
+        self._check()
+
+        # Data list
+        with open(os.path.join(root_dataset, 'lists', image_set + '.txt'), "r") as f:
+            contents = [x.strip() for x in f.readlines()]
+
+        # Load filenames
+        if image_set == 'test' or image_set == 'val':  # Test
+            self.images = [os.path.join(root_dataset, x + '.jpg') for x in contents]
+            if use_gt:
+                self.gt_keypoints = [os.path.join(root_dataset, x + '.lines.txt') for x in contents]
+            self.filenames = [x + '.jpg' for x in contents]
+            if root_keypoint is not None:
+                self.keypoints = [os.path.join(root_keypoint, x + '.lines.txt') for x in contents]
+        else:  # Train
+            self.images = [os.path.join(root_dataset, x[:x.find(' ')] + '.jpg') for x in contents]
+            if use_gt:
+                self.gt_keypoints = [os.path.join(root_dataset, x[:x.find(' ')] + '.lines.txt') for x in contents]
+            self.filenames = [x[:x.find(' ')] + '.jpg' for x in contents]
+            if root_keypoint is not None:
+                self.keypoints = [os.path.join(root_keypoint, x + '.lines.txt') for x in contents]
+
+        self.make_sub_dirs()
+
+        assert len(self.images) == len(self.gt_keypoints)
+        if self.keypoints is not None:
+            assert len(self.images) == len(self.keypoints)
+
+    def _check(self):
+        # Checks
+        if self.image_set not in ['train', 'val', 'test']:
+            raise ValueError
+        assert self.output_dir != self.root, 'Avoid overwriting your dataset!'

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/datasets/image_folder.py

@@ -0,0 +1,75 @@
+import torchvision
+import os
+from PIL import Image
+
+from ..transforms import functional as F, ToTensor
+from .builder import DATASETS
+from .image_folder_lane_base import ImageFolderLaneBase
+
+
+# Load a directory of images for inference
+@DATASETS.register()
+class ImageFolderDataset(torchvision.datasets.VisionDataset):
+    def __init__(self, root_image, root_output, root_target=None, transforms=None,
+                 target_process_fn=None, image_suffix='', target_suffix=''):
+        super().__init__(root_image, transforms, None, None)
+        self.output_dir = root_output
+        self.filenames = []
+        self.images = []
+        self.targets = None if root_target is None else []
+        self.target_process_fn = target_process_fn
+        for filename in sorted(os.listdir(root_image)):
+            suffix_pos = filename.rfind(image_suffix)
+            if suffix_pos != -1:
+                middle_name = filename[:suffix_pos]
+                self.filenames.append(filename)
+                self.images.append(os.path.join(root_image, filename))
+                if self.targets is not None:
+                    self.targets.append(os.path.join(root_target, middle_name + target_suffix))
+
+    def __getitem__(self, index):
+        # Return transformed image / original image / save filename / label (if exist)
+        img = Image.open(self.images[index]).convert('RGB')
+        filename = os.path.join(self.output_dir, self.filenames[index])
+        original_img = F.to_tensor(img).clone()
+
+        # Transforms
+        if self.transforms is not None:
+            img = self.transforms(img)
+
+        # Process potential target
+        target = None
+        if self.targets is not None:
+            target = self.target_process_fn(self.targets[index])
+
+        return img, original_img, {
+            'filename': filename,
+            'target': target
+        }
+
+    def __len__(self):
+        return len(self.images)
+
+
+# Load a directory of images for lane inference
+@DATASETS.register()
+class ImageFolderLaneDataset(ImageFolderLaneBase):
+    def __init__(self, root_image, root_output, root_keypoint=None, root_gt_keypoint=None, root_mask=None,
+                 transforms=None, keypoint_process_fn=None,
+                 image_suffix='', keypoint_suffix='.txt', gt_keypoint_suffix='.txt', mask_suffix=''):
+        super().__init__(root_image, root_output, transforms, keypoint_process_fn)
+        self.keypoints = None if root_keypoint is None else []
+        self.gt_keypoints = None if root_gt_keypoint is None else []
+        self.masks = None if root_mask is None else []
+        for filename in sorted(os.listdir(root_image)):
+            suffix_pos = filename.rfind(image_suffix)
+            if suffix_pos != -1:
+                middle_name = filename[:suffix_pos]
+                self.filenames.append(filename)
+                self.images.append(os.path.join(root_image, filename))
+                if self.keypoints is not None:
+                    self.keypoints.append(os.path.join(root_keypoint, middle_name + keypoint_suffix))
+                if self.gt_keypoints is not None:
+                    self.gt_keypoints.append(os.path.join(root_gt_keypoint, middle_name + gt_keypoint_suffix))
+                if self.masks is not None:
+                    self.masks.append(os.path.join(root_mask, middle_name + mask_suffix))

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/datasets/image_folder_lane_base.py

@@ -0,0 +1,60 @@
+import torchvision
+import os
+from PIL import Image
+
+from ..transforms import functional as F, ToTensor
+
+
+# Base class for lane image folder datasets (usually for visualizations)
+class ImageFolderLaneBase(torchvision.datasets.VisionDataset):
+    def __init__(self, root=None, root_output=None, transforms=None, keypoint_process_fn=None):
+        super().__init__(root, transforms, None, None)
+        self.output_dir = root_output
+        os.makedirs(self.output_dir, exist_ok=True)
+        self.filenames = []
+        self.images = []
+        self.keypoints = None
+        self.gt_keypoints = None
+        self.masks = None
+        self.keypoint_process_fn = keypoint_process_fn
+
+    def __getitem__(self, index):
+        # Return transformed image / original image / save filename / labels (if exist)
+        img = Image.open(self.images[index]).convert('RGB')
+        filename = os.path.join(self.output_dir, self.filenames[index])
+        original_img = F.to_tensor(img).clone()
+        mask = None
+        if self.masks is not None:
+            w, h = F._get_image_size(img)
+            mask = ToTensor.label_to_tensor(
+                F.resize(Image.open(self.masks[index]), size=[h, w], interpolation=Image.NEAREST)
+            )
+
+        # Transforms
+        if self.transforms is not None:
+            img = self.transforms(img)
+
+        # Process potential target
+        keypoint = None
+        gt_keypoint = None
+        if self.keypoints is not None:
+            keypoint = self.keypoint_process_fn(self.keypoints[index])
+        if self.gt_keypoints is not None:
+            gt_keypoint = self.keypoint_process_fn(self.gt_keypoints[index])
+
+        return img, original_img, {
+            'filename': filename,
+            'keypoint': keypoint,
+            'gt_keypoint': gt_keypoint,
+            'mask': mask
+        }
+
+    def make_sub_dirs(self):
+        # Make sub dirs
+        for f in self.filenames:
+            dir_name = os.path.join(self.output_dir, f[:f.rfind('/')])
+            if not os.path.exists(dir_name):
+                os.makedirs(dir_name)
+
+    def __len__(self):
+        return len(self.images)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/datasets/lane_as_bezier.py

@@ -0,0 +1,206 @@
+import os
+import torch
+import torchvision
+import json
+import numpy as np
+from PIL import Image
+
+from .builder import DATASETS
+from ..curve_utils import BezierSampler, get_valid_points
+
+
+class _BezierLaneDataset(torchvision.datasets.VisionDataset):
+    # BezierLaneNet dataset, includes binary seg labels
+    keypoint_color = [0, 0, 0]
+
+    def __init__(self, root, image_set='train', transforms=None, transform=None, target_transform=None,
+                 order=3, num_sample_points=100, aux_segmentation=False):
+        super().__init__(root, transforms, transform, target_transform)
+        self.aux_segmentation = aux_segmentation
+        self.bezier_sampler = BezierSampler(order=order, num_sample_points=num_sample_points)
+        if image_set == 'valfast':
+            raise NotImplementedError('valfast Not supported yet!')
+        elif image_set == 'test' or image_set == 'val':  # Different format (without lane existence annotations)
+            self.test = 2
+        elif image_set == 'val_train':
+            self.test = 3
+        else:
+            self.test = 0
+
+        self.init_dataset(root)
+
+        if image_set != 'valfast':
+            self.bezier_labels = os.path.join(self.bezier_labels_dir, image_set + '_' + str(order) + '.json')
+        elif image_set == 'valfast':
+            raise ValueError
+
+        self.image_set = image_set
+        self.splits_dir = os.path.join(root, 'lists')
+        self._init_all()
+
+    def init_dataset(self, root):
+        raise NotImplementedError
+
+    def __getitem__(self, index):
+        # Return x (input image) & y (mask image, i.e. pixel-wise supervision) & lane existence (a list),
+        # if not just testing,
+        # else just return input image.
+        img = Image.open(self.images[index]).convert('RGB')
+        if self.test >= 2:
+            target = self.masks[index]
+        else:
+            if self.aux_segmentation:
+                target = {'keypoints': self.beziers[index],
+                          'segmentation_mask': Image.open(self.masks[index])}
+            else:
+                target = {'keypoints': self.beziers[index]}
+
+        # Transforms
+        if self.transforms is not None:
+            img, target = self.transforms(img, target)
+
+        if self.test == 0:
+            target = self._post_process(target)
+
+        return img, target
+
+    def __len__(self):
+        return len(self.images)
+
+    def loader_bezier(self):
+        results = []
+        with open(self.bezier_labels, 'r') as f:
+            results += [json.loads(x.strip()) for x in f.readlines()]
+        beziers = []
+        for lanes in results:
+            temp_lane = []
+            for lane in lanes['bezier_control_points']:
+                temp_cps = []
+                for i in range(0, len(lane), 2):
+                    temp_cps.append([lane[i], lane[i + 1]])
+                temp_lane.append(temp_cps)
+            beziers.append(np.array(temp_lane, dtype=np.float32))
+        return beziers
+
+    def _init_all(self):
+        # Got the lists from 4 datasets to be in the same format
+        data_list = 'train.txt' if self.image_set == 'val_train' else self.image_set + '.txt'
+        split_f = os.path.join(self.splits_dir, data_list)
+        with open(split_f, "r") as f:
+            contents = [x.strip() for x in f.readlines()]
+        if self.test == 2:  # Test
+            self.images = [os.path.join(self.image_dir, x + self.image_suffix) for x in contents]
+            self.masks = [os.path.join(self.output_prefix, x + self.output_suffix) for x in contents]
+        elif self.test == 3:  # Test
+            self.images = [os.path.join(self.image_dir, x[:x.find(' ')] + self.image_suffix) for x in contents]
+            self.masks = [os.path.join(self.output_prefix, x[:x.find(' ')] + self.output_suffix) for x in contents]
+        elif self.test == 1:  # Val
+            self.images = [os.path.join(self.image_dir, x[:x.find(' ')] + self.image_suffix) for x in contents]
+            self.masks = [os.path.join(self.mask_dir, x[:x.find(' ')] + '.png') for x in contents]
+        else:  # Train
+            self.images = [os.path.join(self.image_dir, x[:x.find(' ')] + self.image_suffix) for x in contents]
+            if self.aux_segmentation:
+                self.masks = [os.path.join(self.mask_dir, x[:x.find(' ')] + '.png') for x in contents]
+            self.beziers = self.loader_bezier()
+
+    def _post_process(self, target, ignore_seg_index=255):
+        # Get sample points and delete invalid lines (< 2 points)
+        if target['keypoints'].numel() != 0:  # No-lane cases can be handled in loss computation
+            sample_points = self.bezier_sampler.get_sample_points(target['keypoints'])
+            valid_lanes = get_valid_points(sample_points).sum(dim=-1) >= 2
+            target['keypoints'] = target['keypoints'][valid_lanes]
+            target['sample_points'] = sample_points[valid_lanes]
+        else:
+            target['sample_points'] = torch.tensor([], dtype=target['keypoints'].dtype)
+
+        if 'segmentation_mask' in target.keys():  # Map to binary (0 1 255)
+            positive_mask = (target['segmentation_mask'] > 0) * (target['segmentation_mask'] != ignore_seg_index)
+            target['segmentation_mask'][positive_mask] = 1
+
+        return target
+
+
+# TuSimple
+@DATASETS.register()
+class TuSimpleAsBezier(_BezierLaneDataset):
+    colors = [
+        [0, 0, 0],  # background
+        [255, 0, 255], [0, 255, 0], [0, 0, 255], [255, 0, 0], [255, 255, 0], [0, 255, 255],
+        [0, 0, 0]  # ignore
+    ]
+
+    def init_dataset(self, root):
+        self.image_dir = os.path.join(root, 'clips')
+        self.bezier_labels_dir = os.path.join(root, 'bezier_labels')
+        self.mask_dir = os.path.join(root, 'segGT6')
+        self.output_prefix = 'clips'
+        self.output_suffix = '.jpg'
+        self.image_suffix = '.jpg'
+
+
+# CULane
+@DATASETS.register()
+class CULaneAsBezier(_BezierLaneDataset):
+    colors = [
+        [0, 0, 0],  # background
+        [0, 255, 0], [0, 0, 255], [255, 0, 0], [255, 255, 0],
+        [0, 0, 0]  # ignore
+    ]
+
+    def init_dataset(self, root):
+        self.image_dir = root
+        self.bezier_labels_dir = os.path.join(root, 'bezier_labels')
+        self.mask_dir = os.path.join(root, 'laneseg_label_w16')
+        self.output_prefix = './output'
+        self.output_suffix = '.lines.txt'
+        self.image_suffix = '.jpg'
+        if not os.path.exists(self.output_prefix):
+            os.makedirs(self.output_prefix)
+
+
+# LLAMAS
+@DATASETS.register()
+class LLAMAS_AsBezier(_BezierLaneDataset):
+    colors = [
+        [0, 0, 0],  # background
+        [0, 255, 0], [0, 0, 255], [255, 0, 0], [255, 255, 0],
+        [0, 0, 0]  # ignore
+    ]
+
+    def init_dataset(self, root):
+        self.image_dir = os.path.join(root, 'color_images')
+        self.bezier_labels_dir = os.path.join(root, 'bezier_labels')
+        self.mask_dir = os.path.join(root, 'laneseg_labels')
+        self.output_prefix = './output'
+        self.output_suffix = '.lines.txt'
+        self.image_suffix = '.png'
+        if not os.path.exists(self.output_prefix):
+            os.makedirs(self.output_prefix)
+
+
+# Curvelanes
+@DATASETS.register()
+class Curvelanes_AsBezier(CULaneAsBezier):
+    # TODO: Match formats
+    colors = []
+
+    def _init_all(self):
+        # Got the lists from 4 datasets to be in the same format
+        data_list = 'train.txt' if self.image_set == 'val_train' else self.image_set + '.txt'
+        split_f = os.path.join(self.splits_dir, data_list)
+        with open(split_f, "r") as f:
+            contents = [x.strip() for x in f.readlines()]
+        if self.test == 2:  # Test
+            self.images = [os.path.join(self.image_dir, x + self.image_suffix) for x in contents]
+            self.masks = [os.path.join(self.output_prefix, x + self.output_suffix) for x in contents]
+        elif self.test == 3:  # Test
+            self.images = [os.path.join(self.image_dir, x[:x.find(' ')] + self.image_suffix) for x in contents]
+            self.masks = [os.path.join(self.output_prefix, x[:x.find(' ')] + self.output_suffix) for x in contents]
+        elif self.test == 1:  # Val
+            self.images = [os.path.join(self.image_dir, x[:x.find(' ')] + self.image_suffix) for x in contents]
+            self.masks = [os.path.join(self.mask_dir, x[:x.find(' ')] + '.png') for x in contents]
+        else:  # Train
+            self.images = [os.path.join(self.image_dir, x + self.image_suffix) for x in contents]
+            if self.aux_segmentation:
+                self.masks = [os.path.join(self.mask_dir, x[:x.find(' ')] + '.png') for x in contents]
+            self.beziers = self.loader_bezier()

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/datasets/lane_as_segmentation.py

@@ -0,0 +1,120 @@
+import os
+import torch
+from PIL import Image
+from torchvision.datasets import VisionDataset
+
+from .builder import DATASETS
+
+
+# Lane detection as segmentation
+class _StandardLaneDetectionDataset(VisionDataset):
+    keypoint_color = [0, 0, 0]
+
+    def __init__(self, root, image_set, transforms=None):
+        super().__init__(root, transforms, None, None)
+        if image_set == 'valfast':
+            self.test = 1
+        elif image_set == 'test' or image_set == 'val':  # Different format (without lane existence annotations)
+            self.test = 2
+        else:
+            self.test = 0
+
+        self.init_dataset(root)
+        self.image_set = image_set
+        self.splits_dir = os.path.join(root, 'lists')
+
+        self._init_all()
+
+        assert (len(self.images) == len(self.masks))
+
+    def init_dataset(self, root):
+        raise NotImplementedError
+
+    def __getitem__(self, index):
+        # Return x (input image) & y (mask image, i.e. pixel-wise supervision) & lane existence (a list),
+        # if not just testing,
+        # else just return input image.
+        img = Image.open(self.images[index]).convert('RGB')
+        if self.test == 2:
+            target = self.masks[index]
+        elif self.test == 1:
+            target = Image.open(self.masks[index])
+        else:
+            target = Image.open(self.masks[index])
+            lane_existence = torch.tensor(self.lane_existences[index]).float()
+
+        # Transforms
+        if self.transforms is not None:
+            img, target = self.transforms(img, target)
+        if self.test > 0:
+            return img, target
+        else:
+            return img, target, lane_existence
+
+    def __len__(self):
+        return len(self.images)
+
+    def _init_all(self):
+        # Got the lists from 2 datasets to be in the same format
+        split_f = os.path.join(self.splits_dir, self.image_set + '.txt')
+        with open(split_f, "r") as f:
+            contents = [x.strip() for x in f.readlines()]
+        if self.test == 2:  # Test
+            self.images = [os.path.join(self.image_dir, x + self.image_suffix) for x in contents]
+            self.masks = [os.path.join(self.output_prefix, x + self.output_suffix) for x in contents]
+        elif self.test == 1:  # Val
+            self.images = [os.path.join(self.image_dir, x[:x.find(' ')] + self.image_suffix) for x in contents]
+            self.masks = [os.path.join(self.mask_dir, x[:x.find(' ')] + '.png') for x in contents]
+        else:  # Train
+            self.images = [os.path.join(self.image_dir, x[:x.find(' ')] + self.image_suffix) for x in contents]
+            self.masks = [os.path.join(self.mask_dir, x[:x.find(' ')] + '.png') for x in contents]
+            self.lane_existences = [list(map(int, x[x.find(' '):].split())) for x in contents]
+
+
+# TuSimple
+@DATASETS.register()
+class TuSimpleAsSegmentation(_StandardLaneDetectionDataset):
+    colors = [
+        [0, 0, 0],  # background
+        [255, 0, 255], [0, 255, 0], [0, 0, 255], [255, 0, 0], [255, 255, 0], [0, 255, 255],
+        [0, 0, 0]  # ignore
+    ]
+
+    def init_dataset(self, root):
+        self.image_dir = os.path.join(root, 'clips')
+        self.mask_dir = os.path.join(root, 'segGT6')
+        self.output_prefix = 'clips'
+        self.output_suffix = '.jpg'
+        self.image_suffix = '.jpg'
+
+
+# CULane
+@DATASETS.register()
+class CULaneAsSegmentation(_StandardLaneDetectionDataset):
+    colors = [
+        [0, 0, 0],  # background
+        [0, 255, 0], [0, 0, 255], [255, 0, 0], [255, 255, 0],
+        [0, 0, 0]  # ignore
+    ]
+
+    def init_dataset(self, root):
+        self.image_dir = root
+        self.mask_dir = os.path.join(root, 'laneseg_label_w16')
+        self.output_prefix = './output'
+        self.output_suffix = '.lines.txt'
+        self.image_suffix = '.jpg'
+        if not os.path.exists(self.output_prefix):
+            os.makedirs(self.output_prefix)
+
+
+# LLAMAS
+@DATASETS.register()
+class LLAMAS_AsSegmentation(CULaneAsSegmentation):
+    def init_dataset(self, root):
+        self.image_dir = os.path.join(root, 'color_images')
+        self.mask_dir = os.path.join(root, 'laneseg_labels')
+        self.output_prefix = './output'
+        self.output_suffix = '.lines.txt'
+        self.image_suffix = '.png'
+        if not os.path.exists(self.output_prefix):
+            os.makedirs(self.output_prefix)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/datasets/llamas.py

@@ -0,0 +1,74 @@
+import os
+import pickle
+import numpy as np
+from tqdm import tqdm
+
+from .utils import LaneKeypointDataset
+from .builder import DATASETS
+
+
+# LLAMAS direct loading (similar with culane)
+@DATASETS.register()
+class LLAMAS(LaneKeypointDataset):
+    colors = [
+        [0, 0, 0],  # background
+        [0, 255, 0], [0, 0, 255], [255, 0, 0], [255, 255, 0],
+        [0, 0, 0]  # ignore
+    ]
+
+    def __init__(self, root, image_set, transforms=None, transform=None, target_transform=None,
+                 ppl=417, gap=1, start=300, padding_mask=False):
+        super().__init__(root, transforms, transform, target_transform, ppl, gap, start, padding_mask, image_set)
+
+        self._check()
+
+        self.images_path = os.path.join(root, 'color_images')
+        # Data list
+        with open(os.path.join(root, 'lists', image_set + '.txt'), "r") as f:
+            contents = [x.strip() for x in f.readlines()]
+
+        # Load filenames
+        if image_set == 'test' or image_set == 'val':  # Test
+            self.images = [os.path.join(self.images_path, x + '.png') for x in contents]
+            self.targets = [os.path.join('./output', x + '.lines.txt') for x in contents]
+        else:  # Train
+            self.images = [os.path.join(self.images_path, x[:x.find(' ')] + '.png') for x in contents]
+            self.targets = []
+            print('Loading targets into memory...')
+            processed_file = os.path.join(root, 'train_processed_targets')
+            if os.path.exists(processed_file):
+                with open(processed_file, 'rb') as f:
+                    self.targets = pickle.load(f)
+            else:
+                print('Pre-processing will only be performed for 1 time, please wait ~10 minutes.')
+                for x in tqdm(contents):
+                    with open(os.path.join(self.images_path, x[:x.find(' ')] + '.lines.txt'), 'r') as f:
+                        self.targets.append(self._load_target(f.readlines()))
+                with open(processed_file, 'wb') as f:
+                    pickle.dump(self.targets, f)
+            print('Loading complete.')
+
+        assert len(self.targets) == len(self.images)
+
+    def _load_target(self, lines):
+        # Read file content to lists (file content could be empty or variable number of lanes)
+        target = np.array([[[-2.0, self.start + i * self.gap] for i in range(self.ppl)]
+                           for _ in range(len(lines))], dtype=np.float32)
+        for i in range(len(lines)):  # lines=[] will end this immediately
+            temp = [float(k) for k in lines[i].strip().split(' ')]
+            for j in range(int(len(temp) / 2)):
+                x = temp[2 * j]
+                y = temp[2 * j + 1]
+                target[i][target[i][:, 1] == y] = [x, y]
+
+        return target
+
+    @staticmethod
+    def load_target_xy(lines):
+        # A direct loading of JSON file to a list of N x 2 numpy arrays
+        target = []
+        for line in lines:
+            temp = [float(x) for x in line.strip().split(' ')]
+            target.append(np.array(temp).reshape(-1, 2))
+
+        return target

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/datasets/llamas_vis.py

@@ -0,0 +1,50 @@
+import os
+
+from .image_folder_lane_base import ImageFolderLaneBase
+from .builder import DATASETS
+
+
+# Visualization version of CULane
+@DATASETS.register()
+class LLAMAS_Vis(ImageFolderLaneBase):
+    def __init__(self, root_dataset, root_output, root_keypoint, image_set, transforms=None,
+                 keypoint_process_fn=None, use_gt=True):
+        super().__init__(root_dataset, root_output, transforms, keypoint_process_fn)
+        self.image_set = image_set
+        self.images_path = os.path.join(root_dataset, 'color_images')
+
+        self._check()
+
+        # Data list
+        with open(os.path.join(root_dataset, 'lists', image_set + '.txt'), "r") as f:
+            contents = [x.strip() for x in f.readlines()]
+
+        # Load filenames
+        if image_set == 'test' or image_set == 'val':  # Test
+            self.images = [os.path.join(self.images_path, x + '.png') for x in contents]
+            if use_gt:
+                self.gt_keypoints = [os.path.join(self.images_path, x + '.lines.txt') for x in contents]
+            self.filenames = [x + '.png' for x in contents]
+            if root_keypoint is not None:
+                self.keypoints = [os.path.join(root_keypoint, x.replace("_color_rect", "") + '.lines.txt')
+                                  for x in contents]
+        else:  # Train
+            self.images = [os.path.join(self.images_path, x[:x.find(' ')] + '.png') for x in contents]
+            if use_gt:
+                self.gt_keypoints = [os.path.join(self.images_path, x[:x.find(' ')] + '.lines.txt') for x in contents]
+            self.filenames = [x[:x.find(' ')] + '.png' for x in contents]
+            if root_keypoint is not None:
+                self.keypoints = [os.path.join(root_keypoint, x.replace("_color_rect", "") + '.lines.txt')
+                                  for x in contents]
+
+        self.make_sub_dirs()
+
+        assert len(self.images) == len(self.gt_keypoints)
+        if self.keypoints is not None:
+            assert len(self.images) == len(self.keypoints)
+
+    def _check(self):
+        # Checks
+        if self.image_set not in ['train', 'val', 'test']:
+            raise ValueError
+        assert self.output_dir != self.root, 'Avoid overwriting your dataset!'

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/datasets/segmentation.py

@@ -0,0 +1,154 @@
+import os
+import numpy as np
+from PIL import Image
+from torchvision.datasets import VisionDataset
+
+from .builder import DATASETS
+
+
+# Reimplemented based on torchvision.datasets.VOCSegmentation
+class _StandardSegmentationDataset(VisionDataset):
+    def __init__(self, root, image_set, transforms=None, mask_type='.png'):
+        super().__init__(root, transforms, None, None)
+        self.mask_type = mask_type
+        self.images = self.masks = []
+        self.init_dataset(root, image_set)
+        assert (len(self.images) == len(self.masks))
+
+    def init_dataset(self, root, image_set):
+        raise NotImplementedError
+
+    def __getitem__(self, index):
+        img = Image.open(self.images[index]).convert('RGB')
+        # Return x(input image) & y(mask images as a list)
+        # Supports .png & .npy
+        target = Image.open(self.masks[index]) if '.png' in self.masks[index] else np.load(self.masks[index])
+
+        # Transforms
+        if self.transforms is not None:
+            img, target = self.transforms(img, target)
+
+        return img, target
+
+    def __len__(self):
+        return len(self.images)
+
+
+# VOC
+@DATASETS.register()
+class PASCAL_VOC_Segmentation(_StandardSegmentationDataset):
+    categories = [
+        'Background',
+        'Aeroplane', 'Bicycle', 'Bird', 'Boat',
+        'Bottle', 'Bus', 'Car', 'Cat',
+        'Chair', 'Cow', 'Diningtable', 'Dog',
+        'Horse', 'Motorbike', 'Person', 'Pottedplant',
+        'Sheep', 'Sofa', 'Train', 'Tvmonitor'
+    ]
+
+    colors = [
+        [0, 0, 0],
+        [128, 0, 0], [0, 128, 0], [128, 128, 0], [0, 0, 128],
+        [128, 0, 128], [0, 128, 128], [128, 128, 128], [64, 0, 0],
+        [192, 0, 0], [64, 128, 0], [192, 128, 0], [64, 0, 128],
+        [192, 0, 128], [64, 128, 128], [192, 128, 128], [0, 64, 0],
+        [128, 64, 0], [0, 192, 0], [128, 192, 0], [0, 64, 128],
+        [255, 255, 255]  # last color for ignore
+    ]
+
+    def init_dataset(self, root, image_set):
+        image_dir = os.path.join(root, 'JPEGImages')
+        mask_dir = os.path.join(root, 'SegmentationClassAug')
+        splits_dir = os.path.join(root, 'ImageSets/Segmentation')
+        split_f = os.path.join(splits_dir, image_set + '.txt')
+        with open(split_f, "r") as f:
+            file_names = [x.strip() for x in f.readlines()]
+
+        self.images = [os.path.join(image_dir, x + ".jpg") for x in file_names]
+        self.masks = [os.path.join(mask_dir, x + self.mask_type) for x in file_names]
+
+
+# Cityscapes
+@DATASETS.register()
+class CityscapesSegmentation(_StandardSegmentationDataset):
+    categories = [
+        'road', 'sidewalk', 'building', 'wall',
+        'fence', 'pole', 'traffic light', 'traffic sign',
+        'vegetation', 'terrain', 'sky', 'person',
+        'rider', 'car', 'truck', 'bus',
+        'train', 'motorcycle', 'bicycle'
+    ]
+
+    colors = [
+        [128, 64, 128], [244, 35, 232], [70, 70, 70], [102, 102, 156],
+        [190, 153, 153], [153, 153, 153], [250, 170, 30], [220, 220, 0],
+        [107, 142, 35], [152, 251, 152], [70, 130, 180], [220, 20, 60],
+        [255, 0, 0], [0, 0, 142], [0, 0, 70], [0, 60, 100],
+        [0, 80, 100], [0, 0, 230], [119, 11, 32],
+        [0, 0, 0]  # last color for ignore
+    ]
+
+    cities = [
+        'aachen', 'bremen', 'darmstadt', 'erfurt', 'hanover',
+        'krefeld', 'strasbourg', 'tubingen', 'weimar', 'bochum',
+        'cologne', 'dusseldorf', 'hamburg', 'jena', 'monchengladbach',
+        'stuttgart', 'ulm', 'zurich'
+    ]
+
+    def init_dataset(self, root, image_set):
+        image_dir = os.path.join(root, 'leftImg8bit')
+        mask_dir = os.path.join(root, 'gtFine')
+
+        if image_set == 'val':
+            image_dir = os.path.join(image_dir, image_set)
+            mask_dir = os.path.join(mask_dir, image_set)
+        else:
+            image_dir = os.path.join(image_dir, 'train')
+            mask_dir = os.path.join(mask_dir, 'train')
+
+        # We first generate data lists before all this, so we can do this easier
+        splits_dir = os.path.join(root, 'data_lists')
+        split_f = os.path.join(splits_dir, image_set + '.txt')
+        with open(split_f, "r") as f:
+            file_names = [x.strip() for x in f.readlines()]
+
+        self.images = [os.path.join(image_dir, x + "_leftImg8bit.png") for x in file_names]
+        self.masks = [os.path.join(mask_dir, x + "_gtFine_labelIds" + self.mask_type) for x in file_names]
+
+
+# GTAV
+@DATASETS.register()
+class GTAV_Segmentation(CityscapesSegmentation):
+    cities = None
+
+    def init_dataset(self, root, image_set):
+        image_dir = os.path.join(root, 'images')
+        mask_dir = os.path.join(root, 'labels')
+
+        # We first generate data lists before all this, so we can do this easier
+        splits_dir = os.path.join(root, 'data_lists')
+        split_f = os.path.join(splits_dir, image_set + '.txt')
+        with open(split_f, "r") as f:
+            file_names = [x.strip() for x in f.readlines()]
+
+        self.images = [os.path.join(image_dir, x + ".png") for x in file_names]
+        self.masks = [os.path.join(mask_dir, x + self.mask_type) for x in file_names]
+
+
+# SYNTHIA
+@DATASETS.register()
+class SYNTHIA_Segmentation(CityscapesSegmentation):
+    cities = None
+
+    def init_dataset(self, root, image_set):
+        image_dir = os.path.join(root, 'RGB', image_set)
+        mask_dir = os.path.join(root, 'GT/LABELS_CONVERTED', image_set)
+
+        # We first generate data lists before all this, so we can do this easier
+        splits_dir = os.path.join(root, 'data_lists')
+        split_f = os.path.join(splits_dir, image_set + '.txt')
+        with open(split_f, "r") as f:
+            file_names = [x.strip() for x in f.readlines()]
+
+        self.images = [os.path.join(image_dir, x + ".png") for x in file_names]
+        self.masks = [os.path.join(mask_dir, x + self.mask_type) for x in file_names]

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/datasets/tusimple.py

@@ -0,0 +1,66 @@
+import os
+try:
+    import ujson as json
+except ImportError:
+    import json
+import numpy as np
+from tqdm import tqdm
+
+from .utils import LaneKeypointDataset
+from .builder import DATASETS
+
+
+# TuSimple direct loading
+@DATASETS.register()
+class TuSimple(LaneKeypointDataset):
+    colors = [
+        [0, 0, 0],  # background
+        [0, 255, 0], [0, 0, 255], [255, 0, 0], [255, 255, 0],
+        [0, 0, 0]  # ignore
+    ]
+
+    def __init__(self, root, image_set, transforms=None, transform=None, target_transform=None,
+                 ppl=56, gap=10, start=160, padding_mask=False, is_process=True):
+        super().__init__(root, transforms, transform, target_transform, ppl, gap, start, padding_mask, image_set,
+                         is_process)
+
+        self._check()
+
+        # Data list
+        with open(os.path.join(root, 'lists', image_set + '.txt'), "r") as f:
+            contents = [x.strip() for x in f.readlines()]
+
+        # Load image filenames and lanes
+        if image_set == 'test' or image_set == 'val':  # Test
+            self.images = [os.path.join(root, 'clips', x + '.jpg') for x in contents]
+            self.targets = [os.path.join('clips', x + '.jpg') for x in contents]
+        else:  # Train
+            self.images = [os.path.join(root, 'clips', x[:x.find(' ')] + '.jpg') for x in contents]
+
+            # Load target lanes (small dataset, directly load all of them in the memory)
+            print('Loading targets into memory...')
+            target_files = [os.path.join(root, 'label_data_0313.json'),
+                            os.path.join(root, 'label_data_0601.json')]
+            json_contents = self.concat_jsons(target_files)
+            self.targets = []
+            for i in tqdm(range(len(json_contents))):
+                lines = json_contents[i]['lanes']
+                h_samples = json_contents[i]['h_samples']
+                temp = np.array([[[-2.0, self.start + j * self.gap] for j in range(self.ppl)]
+                                for _ in range(len(lines))], dtype=np.float32)
+                for j in range(len(h_samples)):
+                    for k in range(len(lines)):
+                        temp[k][temp[k][:, 1] == h_samples[j]] = [float(lines[k][j]), h_samples[j]]
+                self.targets.append(temp)
+
+        assert len(self.targets) == len(self.images)
+
+    @staticmethod
+    def concat_jsons(filenames):
+        # Concat tusimple lists in jsons (actually only each line is json)
+        results = []
+        for filename in filenames:
+            with open(filename, 'r') as f:
+                results += [json.loads(x.strip()) for x in f.readlines()]
+
+        return results

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/datasets/tusimple_vis.py

@@ -0,0 +1,80 @@
+import os
+import numpy as np
+from tqdm import tqdm
+
+from .image_folder_lane_base import ImageFolderLaneBase
+from .tusimple import TuSimple
+from .builder import DATASETS
+
+
+def dummy_keypoint_process_fn(label):
+    return label
+
+
+# Visualization version of CULane
+@DATASETS.register()
+class TuSimpleVis(ImageFolderLaneBase):
+    def __init__(self, root_dataset, root_output, keypoint_json, image_set, transforms=None,
+                 keypoint_process_fn=None, use_gt=True):
+        super().__init__(root_dataset, root_output, transforms, dummy_keypoint_process_fn)
+        self.image_set = image_set
+
+        self._check()
+
+        # Data list
+        with open(os.path.join(root_dataset, 'lists', image_set + '.txt'), "r") as f:
+            contents = [x.strip() for x in f.readlines()]
+
+        # Load filenames
+        if image_set == 'test' or image_set == 'val':  # Test
+            self.images = [os.path.join(root_dataset, 'clips', x + '.jpg') for x in contents]
+            self.filenames = [os.path.join('clips', x + '.jpg') for x in contents]
+            if use_gt:
+                self.gt_keypoints = []
+                target_files = [os.path.join(root_dataset, 'label_data_0531.json')]
+                if image_set == 'test':
+                    target_files = [os.path.join(root_dataset, 'test_label.json')]
+                json_contents = TuSimple.concat_jsons(target_files)
+                self.gt_keypoints = self.preload_tusimple_labels(json_contents)
+            if keypoint_json is not None:
+                json_contents = TuSimple.concat_jsons([keypoint_json])
+                self.keypoints = self.preload_tusimple_labels(json_contents)
+        else:  # Train
+            self.images = [os.path.join(root_dataset, 'clips', x[:x.find(' ')] + '.jpg') for x in contents]
+            self.filenames = [os.path.join('clips', x[:x.find(' ')] + '.jpg') for x in contents]
+            if use_gt:
+                self.gt_keypoints = []
+                target_files = [os.path.join(root_dataset, 'label_data_0313.json'),
+                                os.path.join(root_dataset, 'label_data_0601.json')]
+                json_contents = TuSimple.concat_jsons(target_files)
+                self.gt_keypoints = self.preload_tusimple_labels(json_contents)
+            if keypoint_json is not None:
+                json_contents = TuSimple.concat_jsons([keypoint_json])
+                self.keypoints = self.preload_tusimple_labels(json_contents)
+
+        self.make_sub_dirs()
+
+        assert len(self.images) == len(self.gt_keypoints)
+        if self.keypoints is not None:
+            assert len(self.images) == len(self.keypoints)
+
+    def _check(self):
+        # Checks
+        if self.image_set not in ['train', 'val', 'test']:
+            raise ValueError
+        assert self.output_dir != self.root, 'Avoid overwriting your dataset!'
+
+    @staticmethod
+    def preload_tusimple_labels(json_contents):
+        # Load a TuSimple label json's content
+        print('Loading json annotation/prediction...')
+        targets = []
+        for i in tqdm(range(len(json_contents))):
+            lines = json_contents[i]['lanes']
+            h_samples = json_contents[i]['h_samples']
+            temp = []
+            for j in range(len(lines)):
+                temp.append(np.array([[float(x), float(y)] for x, y in zip(lines[j], h_samples)]))
+            targets.append(temp)
+
+        return targets

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/datasets/utils.py

@@ -0,0 +1,139 @@
+import os
+import collections.abc
+import torch
+import torchvision
+from PIL import Image
+from utils.transforms import functional_pil as f_pil
+# from torch._six import container_abcs, string_classes, int_classes
+from torch.utils.data._utils.collate import default_collate_err_msg_format, np_str_obj_array_pattern
+
+
+string_classes = (str, bytes)
+int_classes = int
+container_abcs = collections.abc
+
+
+def dict_collate_fn(batch):
+    # To keep each image's label as separate dictionaries, default pytorch behaviour will stack each key
+    # Only modified one line of the pytorch 1.6.0 default collate function
+
+    elem = batch[0]
+    elem_type = type(elem)
+    if isinstance(elem, torch.Tensor):
+        out = None
+        if torch.utils.data.get_worker_info() is not None:
+            # If we're in a background process, concatenate directly into a
+            # shared memory tensor to avoid an extra copy
+            numel = sum([x.numel() for x in batch])
+            storage = elem.storage()._new_shared(numel)
+            out = elem.new(storage)
+        return torch.stack(batch, 0, out=out)
+    elif elem_type.__module__ == 'numpy' and elem_type.__name__ != 'str_' \
+            and elem_type.__name__ != 'string_':
+        elem = batch[0]
+        if elem_type.__name__ == 'ndarray':
+            # array of string classes and object
+            if np_str_obj_array_pattern.search(elem.dtype.str) is not None:
+                raise TypeError(default_collate_err_msg_format.format(elem.dtype))
+
+            return dict_collate_fn([torch.as_tensor(b) for b in batch])
+        elif elem.shape == ():  # scalars
+            return torch.as_tensor(batch)
+    elif isinstance(elem, float):
+        return torch.tensor(batch, dtype=torch.float64)
+    elif isinstance(elem, int_classes):
+        return torch.tensor(batch)
+    elif isinstance(elem, string_classes):
+        return batch
+    elif isinstance(elem, container_abcs.Mapping):
+        return batch  # !Only modified this line
+    elif isinstance(elem, tuple) and hasattr(elem, '_fields'):  # namedtuple
+        return elem_type(*(dict_collate_fn(samples) for samples in zip(*batch)))
+    elif isinstance(elem, container_abcs.Sequence):
+        # check to make sure that the elements in batch have consistent size
+        it = iter(batch)
+        elem_size = len(next(it))
+        if not all(len(elem) == elem_size for elem in it):
+            raise RuntimeError('each element in list of batch should be of equal size')
+        transposed = zip(*batch)
+        return [dict_collate_fn(samples) for samples in transposed]
+
+    raise TypeError(default_collate_err_msg_format.format(elem_type))
+
+
+def generate_lane_label_dict(target):
+    # target: {'keypoints': Tensor, L x N x 2, ...}
+    # Although non-existent keypoints are marked as (-2, y), it is safer to check with > 0
+
+    # Drop invalid lanes (lanes with less than 2 keypoints are seen as invalid)
+    target['lowers'] = torch.tensor([], dtype=target['keypoints'].dtype)
+    target['uppers'] = torch.tensor([], dtype=target['keypoints'].dtype)
+    target['labels'] = torch.tensor([], dtype=torch.int64)
+    if target['keypoints'].numel() > 0:
+        valid_lanes = (target['keypoints'][:, :, 0] > 0).sum(dim=-1) >= 2
+        target['keypoints'] = target['keypoints'][valid_lanes]
+        if target['keypoints'].numel() > 0:  # Still has lanes
+            # Append lowest & highest y coordinates (coordinates start at top-left corner), labels (all 1)
+            # Looks better than giving MIN values
+            target['lowers'] = torch.stack([l[l[:, 0] > 0][:, 1].max() for l in target['keypoints']])
+            target['uppers'] = torch.stack([l[l[:, 0] > 0][:, 1].min() for l in target['keypoints']])
+            target['labels'] = torch.ones(target['keypoints'].shape[0],
+                                          device=target['keypoints'].device, dtype=torch.int64)
+
+    return target
+
+
+# Lanes as keypoints
+class LaneKeypointDataset(torchvision.datasets.VisionDataset):
+    keypoint_color = [0, 0, 0]
+
+    def __init__(self, root, transforms, transform, target_transform,
+                 ppl, gap, start, padding_mask, image_set, is_process):
+        super().__init__(root, transforms, transform, target_transform)
+        self.ppl = ppl  # Sampled points-per-lane
+        self.gap = gap  # y gap between sample points
+        self.start = start  # y coordinate to start annotation
+        self.padding_mask = padding_mask  # Padding mask for transformer
+        self.process_points = image_set == 'train'  # Add lowest & highest y coordinates, lane class labels
+        self.is_process = is_process
+        self.images = []  # placeholder
+        self.targets = []  # placeholder
+        self.image_set = image_set
+
+    def _check(self):
+        # Checks
+        if not os.path.exists('./output'):
+            os.makedirs('./output')
+        if self.image_set not in ['train', 'val', 'test']:
+            raise ValueError
+
+    def __getitem__(self, index):
+        # Return x (input image) & y (L lane with N coordinates (x, y) as np.array (L x N x 2))
+        # Invalid coordinates are marked by (-2, y)
+        # If just testing,
+        # y is the filename to store prediction
+        img = Image.open(self.images[index]).convert('RGB')
+        if type(self.targets[index]) == str:  # Load as paths
+            target = self.targets[index]
+        else:  # Load as dict
+            target = {'keypoints': self.targets[index]}
+
+        if (self.padding_mask or self.process_points) and type(target) == str:
+            print('Testing does not require target padding_mask or process_point!')
+            raise ValueError
+
+        # Add padding mask
+        if self.padding_mask:
+            target['padding_mask'] = Image.new("L", f_pil._get_image_size(img), 0)
+
+        # Transforms
+        if self.transforms is not None:
+            img, target = self.transforms(img, target)
+
+        if self.process_points and self.is_process:
+            target = generate_lane_label_dict(target)
+
+        return img, target
+
+    def __len__(self):
+        return len(self.images)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/datasets/video.py

@@ -0,0 +1,43 @@
+import torch
+from mmcv import VideoReader
+
+from ..transforms import Compose
+from .builder import DATASETS
+
+
+# Load a video for inference
+@DATASETS.register()
+class VideoLoader(object):
+    def __init__(self, filename, transforms=None, batch_size=1, *args, **kwargs):
+        # Don't need ToTensor here
+        self.transforms = Compose(transforms=[t for t in transforms.transforms if t.__class__.__name__ != 'ToTensor'])
+        self.batch_size = batch_size
+        self.video = VideoReader(filename)
+        self.resolution = self.video.resolution
+        self.fps = self.video.fps
+        self.i = 0
+
+    def __next__(self):
+        # Return transformed images / original images
+        # Numpy can suffer a index OOB
+        if self.i >= len(self):
+            raise StopIteration
+
+        images_numpy = self.video[self.i * self.batch_size: (self.i + 1) * self.batch_size]
+        images = torch.stack([torch.from_numpy(img) for img in images_numpy])
+        images = images[..., [2, 1, 0]].permute(0, 3, 1, 2) / 255.0  # BHWC-rgb uint8 -> BCHW-rgb float
+        original_images = images.clone()
+
+        # Transforms
+        if self.transforms is not None:
+            images = self.transforms(images)
+
+        self.i += 1
+
+        return images, original_images
+
+    def __iter__(self):
+        return self
+
+    def __len__(self):
+        return len(self.video) // self.batch_size

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/ddp_utils.py

@@ -0,0 +1,106 @@
+# Simple wrapper-like utils copied from pytorch/vision
+import os
+import torch
+import torch.distributed as dist
+
+
+def setup_for_distributed(is_master):
+    """
+    This function disables printing when not in master process
+    """
+    import builtins as __builtin__
+    builtin_print = __builtin__.print
+
+    def print(*args, **kwargs):
+        force = kwargs.pop('force', False)
+        if is_master or force:
+            builtin_print(*args, **kwargs)
+
+    __builtin__.print = print
+
+
+def is_dist_avail_and_initialized():
+    if not dist.is_available():
+        return False
+    if not dist.is_initialized():
+        return False
+    return True
+
+
+def get_world_size():
+    if not is_dist_avail_and_initialized():
+        return 1
+    return dist.get_world_size()
+
+
+def get_rank():
+    if not is_dist_avail_and_initialized():
+        return 0
+    return dist.get_rank()
+
+
+def is_main_process():
+    return get_rank() == 0
+
+
+def save_on_master(*args, **kwargs):
+    if is_main_process():
+        torch.save(*args, **kwargs)
+
+
+def reduce_dict(input_dict, average=True, to_item=True):
+    """
+    Args:
+        input_dict (dict): all the values will be reduced
+        average (bool): whether to do average or sum
+        to_item (bool): whether convert tensor to its item (used for logging)
+    Reduce the values in the dictionary from all processes so that all processes
+    have the averaged results. Returns a dict with the same fields as
+    input_dict, after reduction.
+    """
+    world_size = get_world_size()
+    if world_size < 2:
+        return input_dict
+    with torch.no_grad():
+        names = []
+        values = []
+        # sort the keys so that they are consistent across processes
+        for k in sorted(input_dict.keys()):
+            names.append(k)
+            values.append(input_dict[k])
+        values = torch.stack(values, dim=0)
+        dist.all_reduce(values)
+        if average:
+            values /= world_size
+        reduced_dict = {k: v.item() if to_item else v for k, v in zip(names, values)}
+    return reduced_dict
+
+
+def init_distributed_mode(cfg):
+    if cfg['state'] == 0 and cfg['world_size'] > 0:  # Restrict ddp to training
+        if 'RANK' in os.environ and 'WORLD_SIZE' in os.environ:
+            cfg['rank'] = int(os.environ["RANK"])
+            cfg['world_size'] = int(os.environ['WORLD_SIZE'])
+            cfg['gpu'] = int(os.environ['LOCAL_RANK'])
+        elif 'SLURM_PROCID' in os.environ:
+            cfg['rank'] = int(os.environ['SLURM_PROCID'])
+            cfg['gpu'] = cfg['rank'] % torch.cuda.device_count()
+        elif hasattr(cfg, "rank"):
+            pass
+        else:
+            print('Not using distributed mode')
+            cfg['distributed'] = False
+            return
+    else:
+        print('Not using distributed mode')
+        cfg['distributed'] = False
+        return
+
+    cfg['distributed'] = True
+    torch.cuda.set_device(cfg['gpu'])
+    cfg['dist_backend'] = 'nccl'
+    print('| distributed init (rank {}): {}'.format(
+        cfg['rank'], cfg['dist_url']), flush=True)
+    torch.distributed.init_process_group(backend=cfg['dist_backend'], init_method=cfg['dist_url'],
+                                         world_size=cfg['world_size'], rank=cfg['rank'])
+    setup_for_distributed(cfg['rank'] == 0)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/frames_to_video.py

@@ -0,0 +1,42 @@
+# Modified from open-mmlab/mmcv
+import os
+import cv2
+from cv2 import VideoWriter_fourcc
+from mmcv.utils import check_file_exist, track_progress
+
+
+def frames2video(frame_dir,
+                 video_file,
+                 file_names,
+                 fps=20,
+                 fourcc='XVID',
+                 show_progress=True):
+    """Read the frame images from a directory and join them as a video.
+    Args:
+        frame_dir (str): The directory containing video frames.
+        video_file (str): Output filename.
+        file_names (list[str]): Image files
+        fps (float): FPS of the output video.
+        fourcc (str): Fourcc of the output video, this should be compatible
+            with the output file type.
+        show_progress (bool): Whether to show a progress bar.
+    """
+
+    first_file = os.path.join(frame_dir, file_names[0])
+    check_file_exist(first_file, 'The start frame not found: ' + first_file)
+    img = cv2.imread(first_file)
+    height, width = img.shape[:2]
+    resolution = (width, height)
+    vwriter = cv2.VideoWriter(video_file, VideoWriter_fourcc(*fourcc), fps, resolution)
+
+    def write_frame(file_idx):
+        filename = os.path.join(frame_dir, file_names[file_idx])
+        img = cv2.imread(filename)
+        vwriter.write(img)
+
+    if show_progress:
+        track_progress(write_frame, range(len(file_names)))
+    else:
+        for i in range(len(file_names)):
+            write_frame(i)
+    vwriter.release()

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/lane_det_utils.py

@@ -0,0 +1,116 @@
+import cv2
+import torch
+import numpy as np
+try:
+    import ujson as json
+except ImportError:
+    import json
+
+from .models.lane_detection.utils import lane_pruning
+
+
+# Adapted from harryhan618/SCNN_Pytorch
+# Note that in tensors we have indices start from 0 and in annotations coordinates start at 1
+def get_lane(prob_map, gap, ppl, thresh, resize_shape=None, dataset='culane'):
+    """
+    Arguments:
+    ----------
+    prob_map: prob map for single lane, np array size (h, w)
+    resize_shape:  reshape size target, (H, W)
+    Return:
+    ----------
+    coords: x coords bottom up every gap px, 0 for non-exist, in resized shape
+    """
+
+    if resize_shape is None:
+        resize_shape = prob_map.shape
+    h, w = prob_map.shape
+    H, W = resize_shape
+    coords = np.zeros(ppl)
+    for i in range(ppl):
+        if dataset == 'tusimple':  # Annotation start at 10 pixel away from bottom
+            y = int(h - (ppl - i) * gap / H * h)
+        elif dataset in ['culane', 'llamas']:  # Annotation start at bottom
+            y = int(h - i * gap / H * h - 1)  # Same as original SCNN code
+        else:
+            raise ValueError
+        if y < 0:
+            break
+        line = prob_map[y, :]
+        id = np.argmax(line)
+        if line[id] > thresh:
+            coords[i] = int(id / w * W)
+    if (coords > 0).sum() < 2:
+        coords = np.zeros(ppl)
+    return coords
+
+
+# Adapted from harryhan618/SCNN_Pytorch
+def prob_to_lines(seg_pred, exist, resize_shape=None, smooth=True, gap=20, ppl=None, thresh=0.3, dataset='culane'):
+    """
+    Arguments:
+    ----------
+    seg_pred: np.array size (num_classes, h, w)
+    resize_shape:  reshape size target, (H, W)
+    exist:   list of existence, e.g. [0, 1, 1, 0]
+    smooth:  whether to smooth the probability or not
+    gap: y pixel gap for sampling
+    ppl:     how many points for one lane
+    thresh:  probability threshold
+    all_points: Whether to save all sample points or just points predicted as lane
+    Return:
+    ----------
+    coordinates: [x, y] list of lanes, e.g.: [ [[9, 569], [50, 549]] ,[[630, 569], [647, 549]] ]
+    """
+    if resize_shape is None:
+        resize_shape = seg_pred.shape[1:]  # seg_pred (num_classes, h, w)
+    _, h, w = seg_pred.shape
+    H, W = resize_shape
+    coordinates = []
+
+    if ppl is None:
+        ppl = round(H / 2 / gap)
+
+    for i in range(1, seg_pred.shape[0]):
+        prob_map = seg_pred[i, :, :]
+        if exist[i - 1]:
+            if smooth:
+                prob_map = cv2.blur(prob_map, (9, 9), borderType=cv2.BORDER_REPLICATE)
+            coords = get_lane(prob_map, gap, ppl, thresh, resize_shape, dataset=dataset)
+            if coords.sum() == 0:
+                continue
+            if dataset == 'tusimple':  # Invalid sample points need to be included as negative value, e.g. -2
+                coordinates.append([[coords[j], H - (ppl - j) * gap] if coords[j] > 0 else [-2,  H - (ppl - j) * gap]
+                                    for j in range(ppl)])
+            elif dataset in ['culane', 'llamas']:
+                coordinates.append([[coords[j], H - j * gap - 1] for j in range(ppl) if coords[j] > 0])
+            else:
+                raise ValueError
+
+    return coordinates
+
+
+# A unified inference function, for segmentation-based lane detection methods
+@torch.no_grad()
+def lane_as_segmentation_inference(net, inputs, input_sizes, gap, ppl, thresh, dataset, max_lane=0, forward=True):
+    # Assume net and images are on the same device
+    # images: B x C x H x W
+    # Return: a list of lane predictions on each image
+    outputs = net(inputs) if forward else inputs  # Support no forwarding inside this function
+    prob_map = torch.nn.functional.interpolate(outputs['out'], size=input_sizes[0], mode='bilinear',
+                                               align_corners=True).softmax(dim=1)
+    existence_conf = outputs['lane'].sigmoid()
+    existence = existence_conf > 0.5
+    if max_lane != 0:  # Lane max number prior for testing
+        existence, existence_conf = lane_pruning(existence, existence_conf, max_lane=max_lane)
+
+    prob_map = prob_map.cpu().numpy()
+    existence = existence.cpu().numpy()
+
+    # Get coordinates for lanes
+    lane_coordinates = []
+    for j in range(existence.shape[0]):
+        lane_coordinates.append(prob_to_lines(prob_map[j], existence[j], resize_shape=input_sizes[1],
+                                              gap=gap, ppl=ppl, thresh=thresh, dataset=dataset))
+
+    return lane_coordinates

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/losses/__init__.py

@@ -0,0 +1,9 @@
+# Implementation based on pytorch 1.6.0
+from .lane_seg_loss import LaneLoss, SADLoss
+from .hungarian_loss import HungarianLoss
+from .hungarian_bezier_loss import HungarianBezierLoss
+from .weighted_ce_loss import WeightedCrossEntropyLoss
+from .torch_loss import torch_loss
+from .focal_loss import _focal_loss, FocalLoss
+from .laneatt_loss import LaneAttLoss
+from .builder import LOSSES

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/losses/_utils.py

@@ -0,0 +1,24 @@
+import torch
+from torch import Tensor
+from typing import Optional
+from torch.nn import _reduction as _Reduction
+
+
+class _Loss(torch.nn.Module):
+    reduction: str
+
+    def __init__(self, size_average=None, reduce=None, reduction: str = 'mean') -> None:
+        super(_Loss, self).__init__()
+        if size_average is not None or reduce is not None:
+            self.reduction = _Reduction.legacy_get_string(size_average, reduce)
+        else:
+            self.reduction = reduction
+
+
+class WeightedLoss(_Loss):
+    def __init__(self, weight: Optional[Tensor] = None, size_average=None, reduce=None,
+                 reduction: str = 'mean') -> None:
+        super(WeightedLoss, self).__init__(size_average, reduce, reduction)
+        if weight is not None and not isinstance(weight, Tensor):
+            weight = torch.tensor(weight).cuda()
+        self.register_buffer('weight', weight)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/losses/builder.py

@@ -0,0 +1,3 @@
+from ..registry import SimpleRegistry
+
+LOSSES = SimpleRegistry()

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/losses/focal_loss.py

@@ -0,0 +1,151 @@
+from typing import Optional
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+# Source: https://github.com/kornia/kornia/blob/f4f70fefb63287f72bc80cd96df9c061b1cb60dd/kornia/losses/focal.py
+def one_hot(labels: torch.Tensor,
+            num_classes: int,
+            device: Optional[torch.device] = None,
+            dtype: Optional[torch.dtype] = None,
+            eps: Optional[float] = 1e-6) -> torch.Tensor:
+    r"""Converts an integer label x-D tensor to a one-hot (x+1)-D tensor.
+
+    Args:
+        labels (torch.Tensor) : tensor with labels of shape :math:`(N, *)`,
+                                where N is batch size. Each value is an integer
+                                representing correct classification.
+        num_classes (int): number of classes in labels.
+        device (Optional[torch.device]): the desired device of returned tensor.
+         Default: if None, uses the current device for the default tensor type
+         (see torch.set_default_tensor_type()). device will be the CPU for CPU
+         tensor types and the current CUDA device for CUDA tensor types.
+        dtype (Optional[torch.dtype]): the desired data type of returned
+         tensor. Default: if None, infers data type from values.
+
+    Returns:
+        torch.Tensor: the labels in one hot tensor of shape :math:`(N, C, *)`,
+
+    Examples::
+        >>> labels = torch.LongTensor([[[0, 1], [2, 0]]])
+        >>> kornia.losses.one_hot(labels, num_classes=3)
+        tensor([[[[1., 0.],
+                  [0., 1.]],
+                 [[0., 1.],
+                  [0., 0.]],
+                 [[0., 0.],
+                  [1., 0.]]]]
+    """
+    if not torch.is_tensor(labels):
+        raise TypeError("Input labels type is not a torch.Tensor. Got {}".format(type(labels)))
+    if not labels.dtype == torch.int64:
+        raise ValueError("labels must be of the same dtype torch.int64. Got: {}".format(labels.dtype))
+    if num_classes < 1:
+        raise ValueError("The number of classes must be bigger than one." " Got: {}".format(num_classes))
+    shape = labels.shape
+    one_hot = torch.zeros(shape[0], num_classes, *shape[1:], device=device, dtype=dtype)
+    return one_hot.scatter_(1, labels.unsqueeze(1), 1.0) + eps
+
+
+def _focal_loss(input: torch.Tensor,
+                target: torch.Tensor,
+                alpha: float,
+                gamma: float = 2.0,
+                reduction: str = 'none',
+                eps: float = 1e-8) -> torch.Tensor:
+    r"""Function that computes Focal loss.
+
+    See :class:`~kornia.losses.FocalLoss` for details.
+    """
+    if not torch.is_tensor(input):
+        raise TypeError("Input type is not a torch.Tensor. Got {}".format(type(input)))
+
+    if not len(input.shape) >= 2:
+        raise ValueError("Invalid input shape, we expect BxCx*. Got: {}".format(input.shape))
+
+    if input.size(0) != target.size(0):
+        raise ValueError('Expected input batch_size ({}) to match target batch_size ({}).'.format(
+            input.size(0), target.size(0)))
+
+    n = input.size(0)
+    out_size = (n, ) + input.size()[2:]
+    if target.size()[1:] != input.size()[2:]:
+        raise ValueError('Expected target size {}, got {}'.format(out_size, target.size()))
+
+    if not input.device == target.device:
+        raise ValueError("input and target must be in the same device. Got: {} and {}".format(
+            input.device, target.device))
+
+    # compute softmax over the classes axis
+    input_soft: torch.Tensor = F.softmax(input, dim=1) + eps
+
+    # create the labels one hot tensor
+    target_one_hot: torch.Tensor = one_hot(target, num_classes=input.shape[1], device=input.device, dtype=input.dtype)
+
+    # compute the actual focal loss
+    weight = torch.pow(-input_soft + 1., gamma)
+
+    focal = -alpha * weight * torch.log(input_soft)
+    loss_tmp = torch.sum(target_one_hot * focal, dim=1)
+
+    if reduction == 'none':
+        loss = loss_tmp
+    elif reduction == 'mean':
+        loss = torch.mean(loss_tmp)
+    elif reduction == 'sum':
+        loss = torch.sum(loss_tmp)
+    else:
+        raise NotImplementedError("Invalid reduction mode: {}".format(reduction))
+    return loss
+
+
+class FocalLoss(nn.Module):
+    r"""Criterion that computes Focal loss.
+
+    According to [1], the Focal loss is computed as follows:
+
+    .. math::
+
+        \text{FL}(p_t) = -\alpha_t (1 - p_t)^{\gamma} \, \text{log}(p_t)
+
+    where:
+       - :math:`p_t` is the model's estimated probability for each class.
+
+
+    Arguments:
+        alpha (float): Weighting factor :math:`\alpha \in [0, 1]`.
+        gamma (float): Focusing parameter :math:`\gamma >= 0`.
+        reduction (str, optional): Specifies the reduction to apply to the
+         output: ‘none’ | ‘mean’ | ‘sum’. ‘none’: no reduction will be applied,
+         ‘mean’: the sum of the output will be divided by the number of elements
+         in the output, ‘sum’: the output will be summed. Default: ‘none’.
+
+    Shape:
+        - Input: :math:`(N, C, *)` where C = number of classes.
+        - Target: :math:`(N, *)` where each value is
+          :math:`0 ≤ targets[i] ≤ C−1`.
+
+    Examples:
+        >>> N = 5  # num_classes
+        >>> kwargs = {"alpha": 0.5, "gamma": 2.0, "reduction": 'mean'}
+        >>> loss = kornia.losses.FocalLoss(**kwargs)
+        >>> input = torch.randn(1, N, 3, 5, requires_grad=True)
+        >>> target = torch.empty(1, 3, 5, dtype=torch.long).random_(N)
+        >>> output = loss(input, target)
+        >>> output.backward()
+
+    References:
+        [1] https://arxiv.org/abs/1708.02002
+    """
+    def __init__(self, alpha: float, gamma: float = 2.0, reduction: str = 'none') -> None:
+        super(FocalLoss, self).__init__()
+        self.alpha: float = alpha
+        self.gamma: float = gamma
+        self.reduction: str = reduction
+        self.eps: float = 1e-6
+
+    def forward(  # type: ignore
+            self, input: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
+        return _focal_loss(input, target, self.alpha, self.gamma, self.reduction, self.eps)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/losses/hungarian_bezier_loss.py

@@ -0,0 +1,210 @@
+# Copied and modified from facebookresearch/detr
+# Refactored and added comments
+
+
+import torch
+from torch.nn import functional as F
+from scipy.optimize import linear_sum_assignment
+
+from ..ddp_utils import is_dist_avail_and_initialized, get_world_size
+from ..curve_utils import BezierSampler, cubic_bezier_curve_segment, get_valid_points
+from ._utils import WeightedLoss
+from .hungarian_loss import HungarianLoss
+from .builder import LOSSES
+
+
+# TODO: Speed-up Hungarian on GPU with tensors
+class _HungarianMatcher(torch.nn.Module):
+    """This class computes an assignment between the targets and the predictions of the network
+
+    For efficiency reasons, the targets don't include the no_object. Because of this, in general,
+    there are more predictions than targets. In this case, we do a 1-to-1 matching of the best predictions,
+    while the others are un-matched (and thus treated as non-objects).
+    POTO matching, which maximizes the cost matrix.
+    """
+
+    def __init__(self, alpha=0.8, bezier_order=3, num_sample_points=100, k=7):
+        super().__init__()
+        self.k = k
+        self.alpha = alpha
+        self.num_sample_points = num_sample_points
+        self.bezier_sampler = BezierSampler(num_sample_points=num_sample_points, order=bezier_order)
+
+    @torch.no_grad()
+    def forward(self, outputs, targets):
+        # Compute the matrices for an entire batch (computation is all pairs, in a way includes the real loss function)
+        # targets: each target: ['keypoints': L x N x 2]
+        # B: batch size; Q: max lanes per-pred, G: total num ground-truth-lanes
+        B, Q = outputs["logits"].shape
+        target_keypoints = torch.cat([i['keypoints'] for i in targets], dim=0)  # G x N x 2
+        target_sample_points = torch.cat([i['sample_points'] for i in targets], dim=0)  # G x num_sample_points x 2
+
+        # Valid bezier segments
+        target_keypoints = cubic_bezier_curve_segment(target_keypoints, target_sample_points)
+        target_sample_points = self.bezier_sampler.get_sample_points(target_keypoints)
+
+        # target_valid_points = get_valid_points(target_sample_points)  # G x num_sample_points
+        G, N = target_keypoints.shape[:2]
+        out_prob = outputs["logits"].sigmoid()  # B x Q
+        out_lane = outputs['curves']  # B x Q x N x 2
+        sizes = [target['keypoints'].shape[0] for target in targets]
+
+        # 1. Local maxima prior
+        _, max_indices = torch.nn.functional.max_pool1d(out_prob.unsqueeze(1),
+                                                        kernel_size=self.k, stride=1,
+                                                        padding=(self.k - 1) // 2, return_indices=True)
+        max_indices = max_indices.squeeze(1)  # B x Q
+        indices = torch.arange(0, Q, dtype=out_prob.dtype, device=out_prob.device).unsqueeze(0).expand_as(max_indices)
+        local_maxima = (max_indices == indices).flatten().unsqueeze(-1).expand(-1, G)  # BQ x G
+
+        # Safe reshape
+        out_prob = out_prob.flatten()  # BQ
+        out_lane = out_lane.flatten(end_dim=1)  # BQ x N x 2
+
+        # 2. Compute the classification cost. Contrary to the loss, we don't use the NLL,
+        # but approximate it in 1 - prob[target class].
+        # Then 1 can be omitted due to it is only a constant.
+        # For binary classification, it is just prob (understand this prob as objectiveness in OD)
+        cost_label = out_prob.unsqueeze(-1).expand(-1, G)  # BQ x G
+
+        # 3. Compute the curve sampling cost
+        cost_curve = 1 - torch.cdist(self.bezier_sampler.get_sample_points(out_lane).flatten(start_dim=-2),
+                                     target_sample_points.flatten(start_dim=-2),
+                                     p=1) / self.num_sample_points  # BQ x G
+
+        # Bound the cost to [0, 1]
+        cost_curve = cost_curve.clamp(min=0, max=1)
+
+        # Final cost matrix (scipy uses min instead of max)
+        C = local_maxima * cost_label ** (1 - self.alpha) * cost_curve ** self.alpha
+        C = -C.view(B, Q, -1).cpu()
+
+        # Hungarian (weighted) on each image
+        indices = [linear_sum_assignment(c[i]) for i, c in enumerate(C.split(sizes, -1))]
+
+        # Return (pred_indices, target_indices) for each image
+        return [(torch.as_tensor(i, dtype=torch.int64), torch.as_tensor(j, dtype=torch.int64)) for i, j in indices]
+
+
+@LOSSES.register()
+class HungarianBezierLoss(WeightedLoss):
+    def __init__(self, curve_weight=1, label_weight=0.1, seg_weight=0.75, alpha=0.8,
+                 num_sample_points=100, bezier_order=3, weight=None, size_average=None, reduce=None, reduction='mean',
+                 ignore_index=-100, weight_seg=None, k=9):
+        super().__init__(weight, size_average, reduce, reduction)
+        self.curve_weight = curve_weight  # Weight for sampled points' L1 distance error between curves
+        self.label_weight = label_weight  # Weight for classification error
+        self.seg_weight = seg_weight  # Weight for binary segmentation auxiliary task
+        self.weight_seg = weight_seg  # BCE loss weight
+        self.ignore_index = ignore_index
+        self.bezier_sampler = BezierSampler(num_sample_points=num_sample_points, order=bezier_order)
+        self.matcher = _HungarianMatcher(alpha=alpha, num_sample_points=num_sample_points, bezier_order=bezier_order,
+                                         k=k)
+        if self.weight is not None and not isinstance(self.weight, torch.Tensor):
+            self.weight = torch.tensor(self.weight).cuda()
+        if self.weight_seg is not None and not isinstance(self.weight_seg, torch.Tensor):
+            self.weight_seg = torch.tensor(self.weight_seg).cuda()
+        self.register_buffer('pos_weight', self.weight[1] / self.weight[0])
+        self.register_buffer('pos_weight_seg', self.weight_seg[1] / self.weight_seg[0])
+
+    def forward(self, inputs, targets, net):
+        outputs = net(inputs)
+        output_curves = outputs['curves']
+        target_labels = torch.zeros_like(outputs['logits'])
+        target_segmentations = torch.stack([target['segmentation_mask'] for target in targets])
+
+        total_targets = 0
+        for i in targets:
+            total_targets += i['keypoints'].numel()
+
+        # CULane actually can produce a whole batch of no-lane images,
+        # in which case, we just calculate the classification loss
+        if total_targets > 0:
+            # Match
+            indices = self.matcher(outputs=outputs, targets=targets)
+            idx = HungarianLoss.get_src_permutation_idx(indices)
+            output_curves = output_curves[idx]
+
+            # Targets (rearrange each lane in the whole batch)
+            # B x N x ... -> BN x ...
+            target_keypoints = torch.cat([t['keypoints'][i] for t, (_, i) in zip(targets, indices)], dim=0)
+            target_sample_points = torch.cat([t['sample_points'][i] for t, (_, i) in zip(targets, indices)], dim=0)
+
+            # Valid bezier segments
+            target_keypoints = cubic_bezier_curve_segment(target_keypoints, target_sample_points)
+            target_sample_points = self.bezier_sampler.get_sample_points(target_keypoints)
+
+            target_labels[idx] = 1  # Any matched lane has the same label 1
+
+        else:
+            # For DDP
+            target_sample_points = torch.tensor([], dtype=torch.float32, device=output_curves.device)
+
+        target_valid_points = get_valid_points(target_sample_points)
+        # Loss
+        loss_curve = self.point_loss(self.bezier_sampler.get_sample_points(output_curves),
+                                     target_sample_points)
+        loss_label = self.classification_loss(inputs=outputs['logits'], targets=target_labels)
+        loss_seg = self.binary_seg_loss(inputs=outputs['segmentations'], targets=target_segmentations)
+
+        loss = self.label_weight * loss_label + self.curve_weight * loss_curve + self.seg_weight * loss_seg
+
+        return loss, {'training loss': loss, 'loss label': loss_label, 'loss curve': loss_curve,
+                      'loss seg': loss_seg,
+                      'valid portion': target_valid_points.float().mean()}
+
+    def point_loss(self, inputs, targets, valid_points=None):
+        # L1 loss on sample points
+        # inputs/targets: L x N x 2
+        # valid points: L x N
+        if targets.numel() == 0:
+            targets = inputs.clone().detach()
+        loss = F.l1_loss(inputs, targets, reduction='none')
+        if valid_points is not None:
+            loss *= valid_points.unsqueeze(-1)
+            normalizer = valid_points.sum()
+        else:
+            normalizer = targets.shape[0] * targets.shape[1]
+            normalizer = torch.as_tensor([normalizer], dtype=inputs.dtype, device=inputs.device)
+        if self.reduction == 'mean':
+            if is_dist_avail_and_initialized():  # Global normalizer should be same across devices
+                torch.distributed.all_reduce(normalizer)
+            normalizer = torch.clamp(normalizer / get_world_size(), min=1).item()
+            loss = loss.sum() / normalizer
+        elif self.reduction == 'sum':  # Usually not needed, but let's have it anyway
+            loss = loss.sum()
+
+        return loss
+
+    def classification_loss(self, inputs, targets):
+        # Typical classification loss (cross entropy)
+        # No need for permutation, assume target is matched to inputs
+
+        # Negative weight as positive weight
+        return F.binary_cross_entropy_with_logits(inputs.unsqueeze(1), targets.unsqueeze(1), pos_weight=self.pos_weight,
+                                                  reduction=self.reduction) / self.pos_weight
+
+    def binary_seg_loss(self, inputs, targets):
+        # BCE segmentation loss with weighting and ignore index
+        # No relation whatever to matching
+
+        # Process inputs
+        inputs = torch.nn.functional.interpolate(inputs, size=targets.shape[-2:], mode='bilinear', align_corners=True)
+        inputs = inputs.squeeze(1)
+
+        # Process targets
+        valid_map = (targets != self.ignore_index)
+        targets[~valid_map] = 0
+        targets = targets.float()
+
+        # Negative weight as positive weight
+        loss = F.binary_cross_entropy_with_logits(inputs, targets, pos_weight=self.pos_weight_seg,
+                                                  reduction='none') / self.pos_weight_seg
+        loss *= valid_map
+
+        if self.reduction == 'mean':
+            return loss.mean()
+        elif self.reduction == 'sum':
+            return loss.sum()
+        else:
+            return loss

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/losses/hungarian_loss.py

@@ -0,0 +1,187 @@
+# Copied and modified from facebookresearch/detr and liuruijin17/LSTR
+# Refactored and added comments
+# Hungarian loss for LSTR
+import torch
+from torch import Tensor
+from torch.nn import functional as F
+from scipy.optimize import linear_sum_assignment
+
+from ._utils import WeightedLoss
+from ..models.lane_detection import cubic_curve_with_projection
+from ..ddp_utils import is_dist_avail_and_initialized, get_world_size
+from .builder import LOSSES
+
+
+@torch.no_grad()
+def lane_normalize_in_batch(keypoints):
+    # Calculate normalization weights for lanes with different number of valid sample points,
+    # so they can produce loss in a similar scale: rather weird but it is what LSTR did
+    # https://github.com/liuruijin17/LSTR/blob/6044f7b2c5892dba7201c273ee632b4962350223/models/py_utils/matcher.py#L59
+    # keypoints: [..., N, 2], ... means arbitrary number of leading dimensions
+    # No gather/reduce is considered here as in the original implementation
+    valid_points = keypoints[..., 0] > 0
+    norm_weights = (valid_points.sum().float() / valid_points.sum(dim=-1).float()) ** 0.5
+    norm_weights /= norm_weights.max()
+
+    return norm_weights, valid_points  # [...], [..., N]
+
+
+# TODO: Speed-up Hungarian on GPU with tensors
+# Nothing will happen with DDP (for at last we use image-wise results)
+class HungarianMatcher(torch.nn.Module):
+    """This class computes an assignment between the targets and the predictions of the network
+
+    For efficiency reasons, the targets don't include the no_object. Because of this, in general,
+    there are more predictions than targets. In this case, we do a 1-to-1 matching of the best predictions,
+    while the others are un-matched (and thus treated as non-objects).
+    """
+
+    def __init__(self, upper_weight=2, lower_weight=2, curve_weight=5, label_weight=3):
+        super().__init__()
+        self.lower_weight = lower_weight
+        self.upper_weight = upper_weight
+        self.curve_weight = curve_weight
+        self.label_weight = label_weight
+
+    @torch.no_grad()
+    def forward(self, outputs, targets):
+        # Compute the matrices for an entire batch (computation is all pairs, in a way includes the real loss function)
+        # targets: each target: ['keypoints': L x N x 2, 'padding_mask': H x W, 'uppers': L, 'lowers': L, 'labels': L]
+        # B: bs; Q: max lanes per-pred, L: num lanes, N: num keypoints per-lane, G: total num ground-truth-lanes
+        bs, num_queries = outputs["logits"].shape[:2]
+        out_prob = outputs["logits"].softmax(dim=-1)  # BQ x 2
+        out_lane = outputs['curves'].flatten(end_dim=-2)  # BQ x 8
+        target_uppers = torch.cat([i['uppers'] for i in targets])
+        target_lowers = torch.cat([i['lowers'] for i in targets])
+        sizes = [target['labels'].shape[0] for target in targets]
+        num_gt = sum(sizes)
+
+        # 1. Compute the classification cost. Contrary to the loss, we don't use the NLL,
+        # but approximate it in 1 - prob[target class].
+        # Then 1 can be omitted due to it is only a constant.
+        # For binary classification, it is just prob (understand this prob as objectiveness in OD)
+        cost_label = -out_prob[..., 1].unsqueeze(-1).flatten(end_dim=-2).repeat(1, num_gt)  # BQ x G
+
+        # 2. Compute the L1 cost between lowers and uppers
+        cost_upper = torch.cdist(out_lane[:, 0:1], target_uppers.unsqueeze(-1), p=1)  # BQ x G
+        cost_lower = torch.cdist(out_lane[:, 1:2], target_lowers.unsqueeze(-1), p=1)  # BQ x G
+
+        # 3. Compute the curve cost
+        target_keypoints = torch.cat([i['keypoints'] for i in targets], dim=0)  # G x N x 2
+        norm_weights, valid_points = lane_normalize_in_batch(target_keypoints)  # G, G x N
+
+        # Masked torch.cdist(p=1)
+        expand_shape = [bs * num_queries, num_gt, target_keypoints.shape[-2]]  # BQ x G x N
+        coefficients = out_lane[:, 2:].unsqueeze(1).expand(*expand_shape[:-1], -1)  # BQ x G x 6
+        out_x = cubic_curve_with_projection(y=target_keypoints[:, :, 1].unsqueeze(0).expand(expand_shape),
+                                            coefficients=coefficients)  # BQ x G x N
+        cost_curve = ((out_x - target_keypoints[:, :, 0].unsqueeze(0).expand(expand_shape)).abs() *
+                      valid_points.unsqueeze(0).expand(expand_shape)).sum(-1)  # BQ x G
+        cost_curve *= norm_weights  # BQ x G
+
+        # Final cost matrix
+        C = self.label_weight * cost_label + self.curve_weight * cost_curve + \
+            self.lower_weight * cost_lower + self.upper_weight * cost_upper
+        C = C.view(bs, num_queries, -1).cpu()
+
+        # Hungarian (weighted) on each image
+        indices = [linear_sum_assignment(c[i]) for i, c in enumerate(C.split(sizes, -1))]
+
+        # Return (pred_indices, target_indices) for each image
+        return [(torch.as_tensor(i, dtype=torch.int64), torch.as_tensor(j, dtype=torch.int64)) for i, j in indices]
+
+
+# The Hungarian loss for LSTR
+@LOSSES.register()
+class HungarianLoss(WeightedLoss):
+    __constants__ = ['reduction']
+
+    def __init__(self, upper_weight=2, lower_weight=2, curve_weight=5, label_weight=3,
+                 weight=None, size_average=None, reduce=None, reduction='mean'):
+        super(HungarianLoss, self).__init__(weight, size_average, reduce, reduction)
+        self.lower_weight = lower_weight
+        self.upper_weight = upper_weight
+        self.curve_weight = curve_weight
+        self.label_weight = label_weight
+        self.matcher = HungarianMatcher(upper_weight, lower_weight, curve_weight, label_weight)
+
+    @staticmethod
+    def get_src_permutation_idx(indices):
+        # Permute predictions following indices
+        # 2-dim indices: (dim0 indices, dim1 indices)
+        batch_idx = torch.cat([torch.full_like(src, i) for i, (src, _) in enumerate(indices)])
+        image_idx = torch.cat([src for (src, _) in indices])
+
+        return batch_idx, image_idx
+
+    def forward(self, inputs: Tensor, targets: Tensor, net):
+        # Support arbitrary auxiliary losses for transformer-based methods
+        if 'padding_mask' in targets[0].keys():  # For multi-scale training support
+            padding_masks = torch.stack([i['padding_mask'] for i in targets])
+            outputs = net(inputs, padding_masks)
+        else:
+            outputs = net(inputs)
+        loss, log_dict = self.calc_full_loss(outputs=outputs, targets=targets)
+        if 'aux' in outputs:
+            for i in range(len(outputs['aux'])):
+                aux_loss, aux_log_dict = self.calc_full_loss(outputs=outputs['aux'][i], targets=targets)
+                loss += aux_loss
+                for k in list(log_dict):  # list(dict) is needed for Python3, since .keys() does not copy like Python2
+                    log_dict[k + ' aux' + str(i)] = aux_log_dict[k]
+
+        return loss, log_dict
+
+    def calc_full_loss(self, outputs, targets):
+        # Match
+        indices = self.matcher(outputs=outputs, targets=targets)
+        idx = self.get_src_permutation_idx(indices)
+
+        # Targets (rearrange each lane in the whole batch)
+        # B x N x ... -> BN x ...
+        target_lowers = torch.cat([t['lowers'][i] for t, (_, i) in zip(targets, indices)], dim=0)
+        target_uppers = torch.cat([t['uppers'][i] for t, (_, i) in zip(targets, indices)], dim=0)
+        target_keypoints = torch.cat([t['keypoints'][i] for t, (_, i) in zip(targets, indices)], dim=0)
+        target_labels = torch.zeros(outputs['logits'].shape[:-1], dtype=torch.int64, device=outputs['logits'].device)
+        target_labels[idx] = 1  # Any matched lane has the same label 1
+
+        # Loss
+        loss_label = self.classification_loss(inputs=outputs['logits'].permute(0, 2, 1), targets=target_labels)
+        output_curves = outputs['curves'][idx]
+        norm_weights, valid_points = lane_normalize_in_batch(target_keypoints)
+        out_x = cubic_curve_with_projection(coefficients=output_curves[:, 2:],
+                                            y=target_keypoints[:, :, 1].clone().detach())
+        loss_curve = self.point_loss(inputs=out_x, targets=target_keypoints[:, :, 0],
+                                     norm_weights=norm_weights, valid_points=valid_points)
+        loss_upper = self.point_loss(inputs=output_curves[:, 0], targets=target_uppers)
+        loss_lower = self.point_loss(inputs=output_curves[:, 1], targets=target_lowers)
+        loss = self.label_weight * loss_label + self.curve_weight * loss_curve + \
+            self.lower_weight * loss_lower + self.upper_weight * loss_upper
+
+        return loss, {'training loss': loss, 'loss label': loss_label, 'loss curve': loss_curve,
+                      'loss upper': loss_upper, 'loss lower': loss_lower}
+
+    def point_loss(self, inputs: Tensor, targets: Tensor, norm_weights=None, valid_points=None) -> Tensor:
+        # L1 loss on sample points, shouldn't it be direct regression?
+        # Also, loss_lowers and loss_uppers in original LSTR code can be done with this same function
+        # No need for permutation, assume target is matched to inputs
+        # inputs/targets: L x N
+        loss = F.l1_loss(inputs, targets, reduction='none')
+        if norm_weights is not None:  # Weights for each lane
+            loss *= norm_weights.unsqueeze(-1).expand_as(loss)
+        if valid_points is not None:  # Valid points
+            loss = loss[valid_points]
+        if self.reduction == 'mean':
+            normalizer = torch.as_tensor([targets.shape[0]], dtype=inputs.dtype, device=inputs.device)
+            if is_dist_avail_and_initialized():  # Global normalizer should be same across devices
+                torch.distributed.all_reduce(normalizer)
+            normalizer = torch.clamp(normalizer / get_world_size(), min=1).item()
+            loss = loss.sum() / normalizer  # Reduce only by number of curves (not number of points)
+        elif self.reduction == 'sum':  # Usually not needed, but let's have it anyway
+            loss = loss.sum()
+
+        return loss
+
+    def classification_loss(self, inputs: Tensor, targets: Tensor) -> Tensor:
+        # Typical classification loss (cross entropy)
+        # No need for permutation, assume target is matched to inputs
+        return F.cross_entropy(inputs, targets, reduction=self.reduction)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/losses/lane_seg_loss.py

@@ -0,0 +1,50 @@
+import torch
+from torch import Tensor
+from typing import Optional
+from torch.nn import functional as F
+
+from ._utils import WeightedLoss
+from .builder import LOSSES
+
+
+# Typical lane detection loss by binary segmentation (e.g. SCNN)
+@LOSSES.register()
+class LaneLoss(WeightedLoss):
+    __constants__ = ['ignore_index', 'reduction']
+    ignore_index: int
+
+    def __init__(self, existence_weight: float = 0.1, weight: Optional[Tensor] = None, size_average=None,
+                 ignore_index: int = -100, reduce=None, reduction: str = 'mean'):
+        super(LaneLoss, self).__init__(weight, size_average, reduce, reduction)
+        self.ignore_index = ignore_index
+        self.existence_weight = existence_weight
+
+    def forward(self, inputs: Tensor, targets: Tensor, lane_existence: Tensor, net, interp_size):
+        outputs = net(inputs)
+        prob_maps = torch.nn.functional.interpolate(outputs['out'], size=interp_size, mode='bilinear',
+                                                    align_corners=True)
+        targets[targets > lane_existence.shape[-1]] = 255  # Ignore extra lanes
+        segmentation_loss = F.cross_entropy(prob_maps, targets, weight=self.weight,
+                                            ignore_index=self.ignore_index, reduction=self.reduction)
+        existence_loss = F.binary_cross_entropy_with_logits(outputs['lane'], lane_existence,
+                                                            weight=None, pos_weight=None, reduction=self.reduction)
+        total_loss = segmentation_loss + self.existence_weight * existence_loss
+
+        return total_loss, {'training loss': total_loss, 'loss seg': segmentation_loss,
+                            'loss exist': existence_loss}
+
+
+# Loss function for SAD
+@LOSSES.register()
+class SADLoss(WeightedLoss):
+    __constants__ = ['ignore_index', 'reduction']
+    ignore_index: int
+
+    def __init__(self, existence_weight: float = 0.1, weight: Optional[Tensor] = None, size_average=None,
+                 ignore_index: int = -100, reduce=None, reduction: str = 'mean'):
+        super(SADLoss, self).__init__(weight, size_average, reduce, reduction)
+        self.ignore_index = ignore_index
+        self.existence_weight = existence_weight
+
+    def forward(self, inputs: Tensor, targets: Tensor):
+        pass

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/losses/laneatt_loss.py

@@ -0,0 +1,191 @@
+import torch
+from torch import Tensor
+from typing import Optional
+# from torch.nn import functional as F
+
+from ._utils import WeightedLoss
+from .builder import LOSSES
+from .focal_loss import FocalLoss
+from ..ddp_utils import get_world_size
+
+INFINITY = 987654.
+
+
+@LOSSES.register()
+class LaneAttLoss(WeightedLoss):
+    def __init__(self,
+                 cls_weight: float = 10.,
+                 reg_weight: float = 1.,
+                 alpha: float = 0.25,
+                 gamma: float = 2.,
+                 num_strips: int = 72 - 1,
+                 num_offsets: int = 72,
+                 t_pos: float = 15.,
+                 t_neg: float = 20.,
+                 weight: Optional[Tensor] = None,
+                 size_average=None,
+                 reduce=None,
+                 reduction: str = 'mean'):
+        super(LaneAttLoss, self).__init__(weight, size_average, reduce, reduction)
+        self.cls_weight = cls_weight
+        self.reg_weight = reg_weight
+        self.num_strips = num_strips
+        self.num_offsets = num_offsets
+        self.t_pos = t_pos
+        self.t_neg = t_neg
+        self.focal_loss = FocalLoss(alpha=alpha, gamma=gamma)
+        self.smooth_l1_loss = torch.nn.SmoothL1Loss()
+
+    def forward(self, inputs, targets, net):
+        # inputs: batchsize x 3 x img_h x img_w
+        # B: batch size, M: max lane
+        labels = {
+            'offsets': torch.stack([i['offsets'] for i in targets], dim=0),  # B x M x num_offsets
+            'starts': torch.stack([i['starts'] for i in targets], dim=0),  # B x M x 1
+            'lengths': torch.stack([i['lengths'] for i in targets], dim=0),  # B x M x 1
+            'flags': torch.stack([i['flags'] for i in targets], dim=0)  # B x M x 1
+        }
+        batch_size = inputs.shape[0]
+
+        outputs = net(inputs)
+        cls_loss = torch.tensor(0, dtype=torch.float32, device=inputs.device)
+        reg_loss = torch.tensor(0, dtype=torch.float32, device=inputs.device)
+        total_positives = 0
+
+        # TODO: Replace these ridiculous codes with batch computation
+        for i in range(batch_size):
+            # Filter lanes that do not exist (confidence == 0)
+            target = {k: v[i][labels['flags'][i]] for k, v in labels.items()}
+
+            # If there are no targets, all proposals have to be negatives (i.e., 0 confidence)
+            if len(target['offsets']) == 0:
+                cls_target = torch.zeros(outputs['logits'][i].shape[0],
+                                         dtype=torch.long, device=outputs['logits'].device)
+                cls_loss = cls_loss + self.focal_loss(outputs['logits'][i], cls_target).sum()
+                continue
+
+            # Match GT
+            positives_mask, invalid_offsets_mask, negatives_mask, target_positives_indices = \
+                self.match_proposals_with_targets(net.module.anchors.clone() if get_world_size() >= 2
+                                                  else net.anchors.clone(), target)
+            # Get positives & negatives
+            positives = {k: v[i][positives_mask] for k, v in outputs.items()}
+            num_positives = positives['logits'].shape[0]
+            total_positives += num_positives
+            negatives = {k: v[i][negatives_mask] for k, v in outputs.items()}
+            num_negatives = negatives['logits'].shape[0]
+
+            # Handle edge case of no positives found
+            if num_positives == 0:
+                cls_target = torch.zeros(outputs['logits'][i].shape[0],
+                                         dtype=torch.long, device=outputs['logits'].device)
+                cls_loss = cls_loss + self.focal_loss(outputs['logits'][i], cls_target).sum()
+                continue
+
+            # Get classification targets (normal cases)
+            cls_target = torch.zeros(num_positives + num_negatives,
+                                     dtype=torch.long, device=outputs['logits'].device)
+            cls_target[:num_positives] = 1
+            cls_pred = torch.cat([positives['logits'], negatives['logits']], dim=0)
+
+            # Regression loss (including length)
+            reg_pred = torch.cat([positives['lengths'][..., None], positives['offsets']], dim=1)
+            with torch.no_grad():
+                target = {k: v[target_positives_indices] for k, v in target.items()}
+                positive_starts = (positives['starts'] * self.num_strips).round().long()
+                targets_starts = (target['starts'] * self.num_strips).round().long()
+                target['lengths'] -= (positive_starts - targets_starts)
+                all_indices = torch.arange(num_positives, dtype=torch.long)
+                ends = (positive_starts + target['lengths'] - 1).round().long()
+                # length + num_offsets + pad (assignment trick ?)
+                invalid_offsets_mask = torch.zeros((num_positives, 1 + self.num_offsets + 1), dtype=torch.int)
+                invalid_offsets_mask[all_indices, 1 + positive_starts] = 1
+                invalid_offsets_mask[all_indices, 1 + ends + 1] -= 1
+                invalid_offsets_mask = invalid_offsets_mask.cumsum(dim=1) == 0
+                invalid_offsets_mask = invalid_offsets_mask[:, :-1]
+                invalid_offsets_mask[:, 0] = False
+                reg_target = torch.cat([target['lengths'][..., None], target['offsets']], dim=1)
+                reg_target[invalid_offsets_mask] = reg_pred[invalid_offsets_mask]  # apply invalids
+
+            # loss
+            reg_loss += self.smooth_l1_loss(reg_pred, reg_target)
+            cls_loss += self.focal_loss(cls_pred, cls_target).sum() / num_positives
+
+        # Batch mean
+        if self.reduction == 'mean':
+            reg_loss /= batch_size
+            cls_loss /= batch_size
+        elif self.reduction != 'sum':
+            raise NotImplementedError
+
+        total_loss = self.cls_weight * cls_loss + self.reg_weight * reg_loss
+
+        # print(torch.tensor(total_positives))
+        return total_loss, {'total loss': total_loss,
+                            'cls loss': cls_loss,
+                            'reg loss': reg_loss,
+                            'all positives': torch.tensor(total_positives).to(reg_loss.device)}
+
+    @torch.no_grad()
+    def match_proposals_with_targets(self, proposals, labels):
+        # Note: matching is between GT and anchors, not predictions
+        # repeat proposals and targets to generate all combinations
+        num_proposals = proposals.shape[0]
+
+        # Match targets with anchors data format (start len offsets)
+        targets = torch.cat([labels['starts'][..., None], labels['lengths'][..., None], labels['offsets']], dim=1)
+        num_targets = targets.shape[0]
+
+        # pad proposals and target for the valid_offset_mask's trick
+        proposals_pad = proposals.new_zeros(proposals.shape[0], proposals.shape[1] + 1)
+        proposals_pad[:, :-1] = proposals
+        proposals = proposals_pad
+        targets_pad = targets.new_zeros(targets.shape[0], targets.shape[1] + 1)
+        targets_pad[:, :-1] = targets
+        targets = targets_pad
+
+        # repeat interleave [a, b] 2 times gives [a, a, b, b]
+        proposals = torch.repeat_interleave(proposals, num_targets, dim=0)
+        # applying this 2 times on [c, d] gives [c, d, c, d]
+        targets = torch.cat(num_proposals * [targets])
+
+        # get start and the intersection of offsets
+        targets_starts = targets[:, 0] * self.num_strips
+        proposals_starts = proposals[:, 0] * self.num_strips
+        starts = torch.max(targets_starts.float(), proposals_starts).round().long()
+        ends = (targets_starts + targets[:, 1].float() - 1.).round().long()
+        lengths = ends - starts + 1
+        ends[lengths < 0] = starts[lengths < 0] - 1
+        lengths[lengths < 0] = 0  # a negative number here means no intersection, thus no length
+
+        # generate valid offsets mask, which works like this:
+        # start with mask [0, 0, 0, 0, 0]
+        # suppose start = 1
+        # length = 2
+        valid_offsets_mask = targets.new_zeros(targets.shape)
+        all_indices = torch.arange(valid_offsets_mask.shape[0], dtype=torch.long, device=targets.device)
+        #  put a one on index `start`, giving [0, 1, 0, 0, 0]
+        valid_offsets_mask[all_indices, 2 + starts] = 1
+        #  put a -1 on the `end` index, giving [0, 1, 0, -1, 0]
+        valid_offsets_mask[all_indices, 2 + ends + 1] -= 1
+        valid_offsets_mask = valid_offsets_mask.cumsum(dim=1) != 0
+        invalid_offsets_mask = ~valid_offsets_mask
+
+        # compute distance
+        # this compares [ac, ad, bc, bd], i.e., all combinations
+        distances = torch.abs((targets - proposals) * valid_offsets_mask.float()).sum(dim=1) / \
+                    (lengths.float() + 1e-9)  # avoid division by zero
+        distances[lengths == 0] = INFINITY
+        invalid_offsets_mask = invalid_offsets_mask.view(num_proposals, num_targets, invalid_offsets_mask.shape[1])
+        distances = distances.view(num_proposals, num_targets)  # d[i,j] = distance from proposal i to target j
+
+        positives = distances.min(dim=1)[0] < self.t_pos
+        negatives = distances.min(dim=1)[0] > self.t_neg
+
+        if positives.sum() == 0:
+            target_positives_indices = torch.tensor([], device=positives.device, dtype=torch.long)
+        else:
+            target_positives_indices = distances[positives].argmin(dim=1)
+
+        invalid_offsets_mask = invalid_offsets_mask[positives, target_positives_indices]
+        return positives, invalid_offsets_mask[:, :-1], negatives, target_positives_indices

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/losses/torch_loss.py

@@ -0,0 +1,10 @@
+from torch import nn
+
+from .builder import LOSSES
+
+
+@LOSSES.register()
+def torch_loss(torch_loss_class, *args, **kwargs):
+    # A direct mapping
+
+    return getattr(nn, torch_loss_class)(*args, **kwargs)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/losses/weighted_ce_loss.py

@@ -0,0 +1,19 @@
+import torch.nn.functional as F
+
+from ._utils import WeightedLoss
+from .builder import LOSSES
+
+
+# Use a base class that can take list weight instead of Tensor
+@LOSSES.register()
+class WeightedCrossEntropyLoss(WeightedLoss):
+    __constants__ = ['ignore_index', 'reduction']
+
+    def __init__(self, weight=None, size_average=None, ignore_index=-100,
+                 reduce=None, reduction='mean'):
+        super().__init__(weight, size_average, reduce, reduction)
+        self.ignore_index = ignore_index
+
+    def forward(self, inputs, targets):
+        return F.cross_entropy(inputs, targets, weight=self.weight,
+                               ignore_index=self.ignore_index, reduction=self.reduction)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/lr_schedulers/__init__.py

@@ -0,0 +1,6 @@
+from .builder import LR_SCHEDULERS
+from .poly_scheduler import poly_scheduler, epoch_poly_scheduler, poly_scheduler_with_warmup
+from .step_scheduler import step_scheduler
+from .cosine_scheduler_wrapper import CosineAnnealingLRWrapper
+from .torch_scheduler import torch_scheduler
+from .cosine_scheduler_wrapper import CosineAnnealingLRWrapper

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/lr_schedulers/builder.py

@@ -0,0 +1,3 @@
+from ..registry import SimpleRegistry
+
+LR_SCHEDULERS = SimpleRegistry()

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/lr_schedulers/cosine_scheduler_wrapper.py

@@ -0,0 +1,11 @@
+from torch.optim import lr_scheduler
+
+from .builder import LR_SCHEDULERS
+
+
+@LR_SCHEDULERS.register()
+def CosineAnnealingLRWrapper(epochs, len_loader, optimizer):
+    # Wrap it so that len_loader is not required in configs
+    return lr_scheduler.CosineAnnealingLR(
+        optimizer,
+        T_max=len_loader * epochs)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/lr_schedulers/poly_scheduler.py

@@ -0,0 +1,29 @@
+import math
+from torch.optim import lr_scheduler
+
+from .builder import LR_SCHEDULERS
+
+
+@LR_SCHEDULERS.register()
+def epoch_poly_scheduler(epochs, len_loader, optimizer, power=0.9):
+    # Poly scheduler for ERFNet
+    return lr_scheduler.LambdaLR(
+        optimizer,
+        lambda x: (1 - math.floor(x / len_loader) / epochs) ** power)
+
+
+@LR_SCHEDULERS.register()
+def poly_scheduler(epochs, len_loader, optimizer, power=0.9):
+    # Poly scheduler
+    return lr_scheduler.LambdaLR(
+        optimizer,
+        lambda x: (1 - x / (len_loader * epochs)) ** power)
+
+
+@LR_SCHEDULERS.register()
+def poly_scheduler_with_warmup(epochs, len_loader, optimizer, power=0.9, warmup_steps=0, start_lr_ratio=0):
+    # Poly scheduler with warmup, start from start_lr_ratio / lr
+    def f(t):  # PEP8-E731
+        return 1. - ((1. - t / warmup_steps) * (1. - start_lr_ratio)) if t < warmup_steps \
+            else (1 - (t - warmup_steps) / (len_loader * epochs - warmup_steps)) ** power
+    return lr_scheduler.LambdaLR(optimizer, f)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/lr_schedulers/step_scheduler.py

@@ -0,0 +1,10 @@
+from torch.optim import lr_scheduler
+
+from .builder import LR_SCHEDULERS
+
+
+@LR_SCHEDULERS.register()
+def step_scheduler(epochs, len_loader, optimizer, step_ratio=0.9, gamma=0.1):
+    return lr_scheduler.StepLR(
+        optimizer,
+        gamma=gamma, step_size=len_loader * epochs * step_ratio)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/lr_schedulers/torch_scheduler.py

@@ -0,0 +1,10 @@
+from torch.optim import lr_scheduler
+
+from .builder import LR_SCHEDULERS
+
+
+@LR_SCHEDULERS.register()
+def torch_scheduler(torch_optim_class, optimizer, *args, **kwargs):
+    # A direct mapping
+
+    return getattr(lr_scheduler, torch_optim_class)(optimizer, *args, **kwargs)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/__init__.py

@@ -0,0 +1,17 @@
+# Some codes may look like copy-paste, but every line of code is manually checked
+# Implementation reference order (not strict):
+# Paper Content = Official Impl > TorchVision Impl > OpenMMLab Impl > Community Impl
+# If there exists an official improved code version that is different from paper content,
+# probably it will not be used here in PytorchAutoDrive.
+from .builder import MODELS
+from . import segmentation
+from . import lane_detection
+from . import transformer
+from . import common_models
+from .backbone_wrappers import free_resnet_backbone, predefined_resnet_backbone
+from .vgg_encoder import VGG16
+from .mobilenet_v2 import MobileNetV2Encoder
+from .mobilenet_v3 import MobileNetV3Encoder
+from .rep_vgg import RepVggEncoder
+from .swin import SwinTransformer
+from .erfnet_encoder import ERFNetEncoder  # add ERFNetEncoder

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/_utils.py

@@ -0,0 +1,100 @@
+from collections import OrderedDict
+
+import torch
+from torch import nn
+
+
+def is_tracing() -> bool:
+    # https://github.com/pytorch/pytorch/issues/42448
+    if torch.__version__ >= '1.7.0':
+        return torch.jit.is_tracing()
+    elif torch.__version__ >= '1.6.0':
+        return torch._C._is_tracing()
+    else:
+        return False
+
+
+class IntermediateLayerGetter(nn.ModuleDict):
+    """
+    Module wrapper that returns intermediate layers from a model
+
+    It has a strong assumption that the modules have been registered
+    into the model in the same order as they are used.
+    This means that one should **not** reuse the same nn.Module
+    twice in the forward if you want this to work.
+
+    Additionally, it is only able to query submodules that are directly
+    assigned to the model. So if `model` is passed, `model.feature1` can
+    be returned, but not `model.feature1.layer2`.
+
+    Arguments:
+        model (nn.Module): model on which we will extract the features
+        return_layers (Dict[name, new_name]): a dict containing the names
+            of the modules for which the activations will be returned as
+            the key of the dict, and the value of the dict is the name
+            of the returned activation (which the user can specify).
+
+    Examples::
+
+        >>> m = torchvision.models.resnet18(pretrained=True)
+        >>> # extract layer1 and layer3, giving as names `feat1` and feat2`
+        >>> new_m = torchvision.models._utils.IntermediateLayerGetter(m,
+        >>>     {'layer1': 'feat1', 'layer3': 'feat2'})
+        >>> out = new_m(torch.rand(1, 3, 224, 224))
+        >>> print([(k, v.shape) for k, v in out.items()])
+        >>>     [('feat1', torch.Size([1, 64, 56, 56])),
+        >>>      ('feat2', torch.Size([1, 256, 14, 14]))]
+    """
+    def __init__(self, model, return_layers):
+        if not set(return_layers).issubset([name for name, _ in model.named_children()]):
+            raise ValueError("return_layers are not present in model")
+
+        orig_return_layers = return_layers
+        return_layers = {k: v for k, v in return_layers.items()}
+        layers = OrderedDict()
+        for name, module in model.named_children():
+            layers[name] = module
+            if name in return_layers:
+                del return_layers[name]
+            if not return_layers:
+                break
+
+        super(IntermediateLayerGetter, self).__init__(layers)
+        self.return_layers = orig_return_layers
+
+    def forward(self, x):
+        out = OrderedDict()
+        for name, module in self.named_children():
+            x = module(x)
+            if name in self.return_layers:
+                out_name = self.return_layers[name]
+                out[out_name] = x
+        return out
+
+
+# Copied from OpenMMLab.
+# https://github.com/open-mmlab/mmsegmentation/blob/fa8c93e78c1679eee558ced5047cc280adfa1c1d/mmseg/models/utils/make_divisible.py#L2
+def make_divisible(value, divisor, min_value=None, min_ratio=0.9):
+    """Make divisible function.
+    This function rounds the channel number to the nearest value that can be
+    divisible by the divisor. It is taken from the original tf repo. It ensures
+    that all layers have a channel number that is divisible by divisor. It can
+    be seen here: https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py  # noqa
+    Args:
+        value (int): The original channel number.
+        divisor (int): The divisor to fully divide the channel number.
+        min_value (int): The minimum value of the output channel.
+            Default: None, means that the minimum value equal to the divisor.
+        min_ratio (float): The minimum ratio of the rounded channel number to
+            the original channel number. Default: 0.9.
+    Returns:
+        int: The modified output channel number.
+    """
+
+    if min_value is None:
+        min_value = divisor
+    new_value = max(min_value, int(value + divisor / 2) // divisor * divisor)
+    # Make sure that round down does not go down by more than (1-min_ratio).
+    if new_value < min_ratio * value:
+        new_value += divisor
+    return new_value

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/backbone_wrappers.py

@@ -0,0 +1,39 @@
+from . import resnet
+from .resnet import _resnet, Bottleneck, BasicBlock
+from .builder import MODELS
+from ._utils import IntermediateLayerGetter
+
+
+# Maybe it is safe to assume no new block types
+block_map = {
+    'BasicBlock': BasicBlock,
+    'BottleNeck': Bottleneck
+}
+
+
+def parse_return_layers(return_layer):
+    if isinstance(return_layer, str):
+        return {return_layer: 'out'}
+    elif isinstance(return_layer, dict):
+        return return_layer
+    else:
+        raise TypeError('return_layer can either be direct dict or a string, not {}'.format(type(return_layer)))
+
+
+@MODELS.register()
+def free_resnet_backbone(arch, block, layers, pretrained, return_layer, **kwargs):
+    block = block_map[block]
+    net = _resnet(arch, block, layers, pretrained, progress=True, **kwargs)
+    return_layers = parse_return_layers(return_layer)
+
+    return IntermediateLayerGetter(net, return_layers=return_layers)
+
+
+@MODELS.register()
+def predefined_resnet_backbone(backbone_name, return_layer, **kwargs):
+    backbone = resnet.__dict__[backbone_name](**kwargs)
+    return_layers = parse_return_layers(return_layer)
+
+    return IntermediateLayerGetter(backbone, return_layers=return_layers)
+
+

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/builder.py

@@ -0,0 +1,3 @@
+from ..registry import SimpleRegistry
+
+MODELS = SimpleRegistry()

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/__init__.py

@@ -0,0 +1,3 @@
+from .blocks import *
+from .heads import *
+from .plugins import *

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/blocks/__init__.py

@@ -0,0 +1,3 @@
+from .inverted_residual import InvertedResidual, InvertedResidualV3
+from .non_bottleneck_1d import non_bottleneck_1d
+from .dilated_bottleneck import DilatedBottleneck

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/blocks/dilated_bottleneck.py

@@ -0,0 +1,42 @@
+import torch
+import torch.nn as nn
+
+from ...builder import MODELS
+
+
+@MODELS.register()
+class DilatedBottleneck(nn.Module):
+    # Refactored from https://github.com/chensnathan/YOLOF/blob/master/yolof/modeling/encoder.py
+    # Diff from typical ResNetV1.5 BottleNeck:
+    # flexible expansion rate, forbids downsampling, relu immediately after last conv
+    def __init__(self,
+                 in_channels=512,
+                 mid_channels=128,
+                 dilation=1):
+        super().__init__()
+        self.conv1 = nn.Sequential(
+            nn.Conv2d(in_channels, mid_channels, kernel_size=1, padding=0),
+            nn.BatchNorm2d(mid_channels)
+        )
+        self.conv2 = nn.Sequential(
+            nn.Conv2d(mid_channels, mid_channels,
+                      kernel_size=3, padding=dilation, dilation=dilation),
+            nn.BatchNorm2d(mid_channels)
+        )
+        self.conv3 = nn.Sequential(
+            nn.Conv2d(mid_channels, in_channels, kernel_size=1, padding=0),
+            nn.BatchNorm2d(in_channels)
+        )
+        self.relu = nn.ReLU(inplace=True)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        identity = x
+        out = self.conv1(x)
+        out = self.relu(out)
+        out = self.conv2(out)
+        out = self.relu(out)
+        out = self.conv3(out)
+        out = self.relu(out)
+        out = out + identity
+
+        return out

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/blocks/inverted_residual.py

@@ -0,0 +1,130 @@
+from torch import nn as nn
+
+from ..plugins import SELayer
+
+
+class InvertedResidual(nn.Module):
+    """InvertedResidual block for MobileNetV2.
+    Args:
+        in_channels (int): The input channels of the InvertedResidual block.
+        out_channels (int): The output channels of the InvertedResidual block.
+        stride (int): Stride of the middle (first) 3x3 convolution.
+        expand_ratio (int): Adjusts number of channels of the hidden layer
+            in InvertedResidual by this amount.
+        dilation (int): Dilation rate of depthwise conv. Default: 1
+    Returns:
+        Tensor: The output tensor.
+    """
+
+    def __init__(self, in_channels, out_channels, stride, expand_ratio, dilation=1, bias=False):
+        super(InvertedResidual, self).__init__()
+        self.stride = stride
+        assert stride in [1, 2], f'stride must in [1, 2]. ' \
+                                 f'But received {stride}.'
+        self.use_res_connect = self.stride == 1 and in_channels == out_channels
+        hidden_dim = int(round(in_channels * expand_ratio))
+        layers = []
+        if expand_ratio != 1:
+            layers.extend([
+                nn.Conv2d(in_channels=in_channels, out_channels=hidden_dim, kernel_size=1, bias=bias),
+                nn.BatchNorm2d(hidden_dim),
+                nn.ReLU6()  # min(max(0, x), 6)
+            ])
+
+        layers.extend([
+            nn.Conv2d(in_channels=hidden_dim, out_channels=hidden_dim, kernel_size=3, stride=stride,
+                      padding=dilation, dilation=dilation, groups=hidden_dim, bias=bias),
+            nn.BatchNorm2d(hidden_dim),
+            nn.ReLU6(),
+            nn.Conv2d(in_channels=hidden_dim, out_channels=out_channels, kernel_size=1, bias=bias),
+            nn.BatchNorm2d(out_channels)
+        ])
+        self.conv = nn.Sequential(*layers)
+
+    def forward(self, x):
+
+        def _inner_forward(x):
+            if self.use_res_connect:
+                return x + self.conv(x)
+            else:
+                return self.conv(x)
+
+        out = _inner_forward(x)
+
+        return out
+
+
+class InvertedResidualV3(nn.Module):
+    """Inverted Residual Block for MobileNetV3.
+    Args:
+        in_channels (int): The input channels of this Module.
+        out_channels (int): The output channels of this Module.
+        mid_channels (int): The input channels of the depthwise convolution.
+        kernel_size (int): The kernel size of the depthwise convolution. Default: 3.
+        stride (int): The stride of the depthwise convolution. Default: 1.
+        with_se (dict): with or without se layer. Default: False, which means no se layer.
+        with_expand_conv (bool): Use expand conv or not. If set False,
+            mid_channels must be the same with in_channels. Default: True.
+        act_cfg (dict): Config dict for activation layer.
+            Default: dict(type='ReLU').
+    Returns:
+        Tensor: The output tensor.
+    """
+
+    def __init__(self, in_channels, out_channels, mid_channels, kernel_size=3, stride=1, with_se=False,
+                 with_expand_conv=True, act='HSwish', bias=False, dilation=1):
+        super(InvertedResidualV3, self).__init__()
+        self.with_res_shortcut = (stride == 1 and in_channels == out_channels)
+        assert stride in [1, 2]
+        activation_layer = nn.Hardswish if act == 'HSwish' else nn.ReLU6
+        self.with_se = with_se
+        self.with_expand_conv = with_expand_conv
+        if not self.with_expand_conv:
+            assert mid_channels == in_channels
+        if self.with_expand_conv:
+            self.expand_conv = nn.Sequential(
+                nn.Conv2d(in_channels=in_channels, out_channels=mid_channels, kernel_size=1, stride=1, padding=0,
+                          bias=bias),
+                nn.BatchNorm2d(mid_channels),
+                activation_layer()
+            )
+        if stride > 1 and dilation > 1:
+            raise ValueError('Can\'t have stride and dilation both > 1 in MobileNetV3')
+        self.depthwise_conv = nn.Sequential(
+            nn.Conv2d(in_channels=mid_channels, out_channels=mid_channels, kernel_size=kernel_size, stride=stride,
+                      padding=(kernel_size - 1) // 2 * dilation, dilation=dilation, groups=mid_channels, bias=bias),
+            nn.BatchNorm2d(mid_channels),
+            activation_layer()
+        )
+        if self.with_se:
+            self.se = SELayer(channels=mid_channels, ratio=4)
+
+        self.linear_conv = nn.Sequential(
+            nn.Conv2d(in_channels=mid_channels, out_channels=out_channels, kernel_size=1, stride=1, padding=0,
+                      bias=False),
+            nn.BatchNorm2d(out_channels)
+        )
+
+    def forward(self, x):
+
+        def _inner_forward(x):
+            out = x
+
+            if self.with_expand_conv:
+                out = self.expand_conv(out)
+
+            out = self.depthwise_conv(out)
+
+            if self.with_se:
+                out = self.se(out)
+
+            out = self.linear_conv(out)
+
+            if self.with_res_shortcut:
+                return x + out
+            else:
+                return out
+
+        out = _inner_forward(x)
+
+        return out

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/blocks/non_bottleneck_1d.py

@@ -0,0 +1,34 @@
+# ERFNet's residual block
+from torch import nn as nn
+from torch.nn import functional as F
+
+
+class non_bottleneck_1d(nn.Module):
+    def __init__(self, chann, dropprob, dilated):
+        super().__init__()
+        self.conv3x1_1 = nn.Conv2d(chann, chann, (3, 1), stride=1, padding=(1, 0), bias=True)
+        self.conv1x3_1 = nn.Conv2d(chann, chann, (1, 3), stride=1, padding=(0, 1), bias=True)
+        self.bn1 = nn.BatchNorm2d(chann, eps=1e-03)
+        self.conv3x1_2 = nn.Conv2d(chann, chann, (3, 1), stride=1, padding=(1 * dilated, 0),
+                                   bias=True, dilation=(dilated, 1))
+        self.conv1x3_2 = nn.Conv2d(chann, chann, (1, 3), stride=1, padding=(0, 1 * dilated),
+                                   bias=True, dilation=(1, dilated))
+        self.bn2 = nn.BatchNorm2d(chann, eps=1e-03)
+        self.dropout = nn.Dropout2d(dropprob)
+
+    def forward(self, input):
+        output = self.conv3x1_1(input)
+        output = F.relu(output)
+        output = self.conv1x3_1(output)
+        output = self.bn1(output)
+        output = F.relu(output)
+
+        output = self.conv3x1_2(output)
+        output = F.relu(output)
+        output = self.conv1x3_2(output)
+        output = self.bn2(output)
+
+        if self.dropout.p != 0:
+            output = self.dropout(output)
+
+        return F.relu(output + input)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/heads/__init__.py

@@ -0,0 +1,7 @@
+from .busd import BUSD
+from .scnn_decoder import SCNNDecoder
+from .plain_decoder import PlainDecoder
+from .simple_lane_exist import SimpleLaneExist
+from .encoder_decoder_lane_exist import EDLaneExist
+from .resa_lane_exist import RESALaneExist
+from .uper_head import UperHead

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/heads/busd.py

@@ -0,0 +1,57 @@
+from torch import nn as nn
+from torch.nn import functional as F
+
+from ...builder import MODELS
+from ..blocks import non_bottleneck_1d
+
+
+class BilateralUpsamplerBlock(nn.Module):
+    # Added a coarse path to the original ERFNet UpsamplerBlock
+    # Copied and modified from:
+    # https://github.com/ZJULearning/resa/blob/14b0fea6a1ab4f45d8f9f22fb110c1b3e53cf12e/models/decoder.py#L67
+
+    def __init__(self, ninput, noutput):
+        super(BilateralUpsamplerBlock, self).__init__()
+        self.conv = nn.ConvTranspose2d(ninput, noutput, 3, stride=2, padding=1, output_padding=1, bias=True)
+        self.bn = nn.BatchNorm2d(noutput, eps=1e-3, track_running_stats=True)
+        self.follows = nn.ModuleList(non_bottleneck_1d(noutput, 0, 1) for _ in range(2))
+
+        # interpolate
+        self.interpolate_conv = nn.Conv2d(ninput, noutput, kernel_size=1, bias=False)
+        self.interpolate_bn = nn.BatchNorm2d(noutput, eps=1e-3)
+
+    def forward(self, input):
+        # Fine branch
+        output = self.conv(input)
+        output = self.bn(output)
+        out = F.relu(output)
+        for follow in self.follows:
+            out = follow(out)
+
+        # Coarse branch (keep at align_corners=True)
+        interpolate_output = self.interpolate_conv(input)
+        interpolate_output = self.interpolate_bn(interpolate_output)
+        interpolate_output = F.relu(interpolate_output)
+        interpolated = F.interpolate(interpolate_output, size=out.shape[-2:], mode='bilinear', align_corners=True)
+
+        return out + interpolated
+
+
+@MODELS.register()
+class BUSD(nn.Module):
+    # Bilateral Up-Sampling Decoder in RESA paper,
+    # make it work for arbitrary input channels (8x up-sample then predict).
+    # Drops transposed prediction layer in ERFNet, while adds an extra up-sampling block.
+
+    def __init__(self, in_channels=128, num_classes=5):
+        super(BUSD, self).__init__()
+        base = in_channels // 8
+        self.layers = nn.ModuleList(BilateralUpsamplerBlock(ninput=base * 2 ** (3 - i), noutput=base * 2 ** (2 - i))
+                                    for i in range(3))
+        self.output_proj = nn.Conv2d(base, num_classes, kernel_size=1, bias=True)  # Keep bias=True for prediction
+
+    def forward(self, x):
+        for layer in self.layers:
+            x = layer(x)
+
+        return self.output_proj(x)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/heads/encoder_decoder_lane_exist.py

@@ -0,0 +1,50 @@
+from torch import nn as nn
+from torch.nn import functional as F
+
+from ...builder import MODELS
+
+
+@MODELS.register()
+class EDLaneExist(nn.Module):
+    # Lane exist head for ERFNet, ENet
+    # Really tricky without global pooling
+
+    def __init__(self, num_output, flattened_size=3965, dropout=0.1, pool='avg'):
+        super().__init__()
+
+        self.layers = nn.ModuleList()
+        self.layers.append(nn.Conv2d(128, 32, (3, 3), stride=1, padding=(4, 4), bias=False, dilation=(4, 4)))
+        self.layers.append(nn.BatchNorm2d(32, eps=1e-03))
+
+        self.layers_final = nn.ModuleList()
+        self.layers_final.append(nn.Dropout2d(dropout))
+        self.layers_final.append(nn.Conv2d(32, 5, (1, 1), stride=1, padding=(0, 0), bias=True))
+
+        if pool == 'max':
+            self.pool = nn.MaxPool2d(2, stride=2)
+        elif pool == 'avg':
+            self.pool = nn.AvgPool2d(2, stride=2)
+        else:
+            raise RuntimeError("This type of pool has not been defined yet!")
+
+        self.linear1 = nn.Linear(flattened_size, 128)
+        self.linear2 = nn.Linear(128, num_output)
+
+    def forward(self, input):
+        output = input
+        for layer in self.layers:
+            output = layer(output)
+
+        output = F.relu(output)
+
+        for layer in self.layers_final:
+            output = layer(output)
+
+        output = F.softmax(output, dim=1)
+        output = self.pool(output)
+        output = output.flatten(start_dim=1)
+        output = self.linear1(output)
+        output = F.relu(output)
+        output = self.linear2(output)
+
+        return output

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/heads/plain_decoder.py

@@ -0,0 +1,19 @@
+from torch import nn as nn
+
+from ...builder import MODELS
+
+
+@MODELS.register()
+class PlainDecoder(nn.Module):
+    # Plain decoder (albeit simplest) from the RESA paper
+
+    def __init__(self, in_channels=128, num_classes=5):
+        super(PlainDecoder, self).__init__()
+        self.dropout1 = nn.Dropout2d(0.1)
+        self.conv1 = nn.Conv2d(in_channels, num_classes, 1, bias=True)
+
+    def forward(self, x):
+        x = self.dropout1(x)
+        x = self.conv1(x)
+
+        return x

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/heads/resa_lane_exist.py

@@ -0,0 +1,30 @@
+from torch import nn as nn
+from torch.nn import functional as F
+
+from ...builder import MODELS
+
+
+@MODELS.register()
+class RESALaneExist(nn.Module):
+    def __init__(self, num_output, flattened_size=3965, dropout=0.1, in_channels=128):
+        super().__init__()
+
+        self.layers = nn.ModuleList()
+        self.layers.append(nn.Dropout2d(dropout))
+        self.layers.append(nn.Conv2d(in_channels, num_output + 1, (1, 1), stride=1, padding=(0, 0), bias=True))
+        self.pool = nn.AvgPool2d(2, stride=2)
+        self.linear1 = nn.Linear(flattened_size, 128)
+        self.linear2 = nn.Linear(128, num_output)
+
+    def forward(self, input):
+        output = input
+        for layer in self.layers:
+            output = layer(output)
+        output = F.softmax(output, dim=1)
+        output = self.pool(output)
+        output = output.flatten(start_dim=1)
+        output = self.linear1(output)
+        output = F.relu(output)
+        output = self.linear2(output)
+
+        return output

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/heads/scnn_decoder.py

@@ -0,0 +1,32 @@
+from torch import nn as nn
+from torch.nn import functional as F
+
+from ...builder import MODELS
+
+
+@MODELS.register()
+class SCNNDecoder(nn.Module):
+    # Unused
+    # SCNN original decoder for ResNet-101, very large channels, maybe impossible to add anything
+    # resnet-101 -> H x W x 2048
+    # 3x3 Conv -> H x W x 512
+    # Dropout 0.1
+    # 1x1 Conv -> H x W x 5
+    # https://github.com/XingangPan/SCNN/issues/35
+
+    def __init__(self, in_channels=2048, num_classes=5):
+        super(SCNNDecoder, self).__init__()
+        out_channels = in_channels // 4
+        self.conv1 = nn.Conv2d(in_channels, out_channels, 3, padding=1, bias=False)
+        self.bn1 = nn.BatchNorm2d(out_channels)
+        self.dropout1 = nn.Dropout2d(0.1)
+        self.conv2 = nn.Conv2d(out_channels, num_classes, 1, bias=False)
+
+    def forward(self, x):
+        x = self.conv1(x)
+        x = self.bn1(x)
+        x = F.relu(x)
+        x = self.dropout1(x)
+        x = self.conv2(x)
+
+        return x

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/heads/simple_lane_exist.py

@@ -0,0 +1,28 @@
+import torch
+from torch import nn as nn
+from torch.nn import functional as F
+
+from ...builder import MODELS
+
+
+@MODELS.register()
+class SimpleLaneExist(nn.Module):
+    # Typical lane existence head originated from the SCNN paper
+
+    def __init__(self, num_output, flattened_size=4500):
+        super().__init__()
+        self.avgpool = nn.AvgPool2d(2, 2)
+        self.linear1 = nn.Linear(flattened_size, 128)
+        self.linear2 = nn.Linear(128, num_output)
+
+    def forward(self, input, predict=False):
+        # input: logits
+        output = self.avgpool(input)
+        output = output.flatten(start_dim=1)
+        output = self.linear1(output)
+        output = F.relu(output)
+        output = self.linear2(output)
+        if predict:
+            output = torch.sigmoid(output)
+
+        return output

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/heads/uper_head.py

@@ -0,0 +1,77 @@
+import torch
+import torch.nn as nn
+from torch.nn import functional as F
+
+from ...builder import MODELS
+from ..plugins.ppm import PPM
+
+
+@MODELS.register()
+class UperHead(nn.Module):
+    def __init__(self, in_channels, channels, pool_scales=(1, 2, 3, 6), align_corners=False):
+        super(UperHead, self).__init__()
+        self.in_channels = in_channels
+        self.channels = channels
+        self.align_corners = align_corners
+        # PSP module
+        self.psp_modules = PPM(pool_scales=pool_scales, in_channels=self.in_channels[-1], channels=self.channels,
+                               align_corners=align_corners)
+        self.psp_bottleneck = nn.Sequential(
+            nn.Conv2d(in_channels=self.in_channels[-1] + len(pool_scales) * self.channels, out_channels=self.channels,
+                      kernel_size=3, padding=1),
+            nn.BatchNorm2d(self.channels),
+            nn.ReLU()
+        )
+        # FPN module
+        self.lateral_convs = nn.ModuleList()
+        self.fpn_convs = nn.ModuleList()
+        for in_channel in self.in_channels[:-1]:
+            lateral_conv = nn.Sequential(
+                nn.Conv2d(in_channel, self.channels, 1),
+                nn.BatchNorm2d(self.channels),
+                nn.ReLU())
+            fpn_conv = nn.Sequential(
+                nn.Conv2d(self.channels, self.channels, 3, padding=1),
+                nn.BatchNorm2d(self.channels),
+                nn.ReLU()
+            )
+            self.lateral_convs.append(lateral_conv)
+            self.fpn_convs.append(fpn_conv)
+        self.fpn_bottleneck = nn.Sequential(
+            nn.Conv2d(len(self.in_channels) * self.channels, self.channels, 3, padding=1),
+            nn.BatchNorm2d(self.channels),
+            nn.ReLU()
+        )
+
+    def psp_forward(self, inputs):
+        # forward function for psp module
+        x = inputs[-1]
+        psp_outs = [x]
+        psp_outs.extend(self.psp_modules(x))
+        psp_outs = torch.cat(psp_outs, dim=1)
+        outputs = self.psp_bottleneck(psp_outs)
+
+        return outputs
+
+    def forward(self, inputs):
+        assert isinstance(inputs, tuple), 'inputs must be a tuple'
+        inputs = list(inputs)
+        # build laterals
+        laterals = [lateral_conv(inputs[i]) for i, lateral_conv in enumerate(self.lateral_convs)]
+        laterals.append(self.psp_forward(inputs))
+        # build top-down path
+        used_backbone_levels = len(laterals)
+        for i in range(used_backbone_levels - 1, 0, -1):
+            prev_shape = laterals[i - 1].shape[2:]
+            laterals[i - 1] = laterals[i - 1] + F.interpolate(laterals[i], size=prev_shape, mode='bilinear',
+                                                              align_corners=self.align_corners)
+        # build outputs
+        fpn_outs = [self.fpn_convs[i](laterals[i]) for i in range(used_backbone_levels - 1)]
+        # add psp feature
+        fpn_outs.append(laterals[-1])
+        for i in range(used_backbone_levels - 1, 0, -1):
+            fpn_outs[i] = F.interpolate(fpn_outs[i], fpn_outs[0].shape[2:], mode='bilinear',
+                                       align_corners=self.align_corners)
+        fpn_outs = torch.cat(fpn_outs, dim=1)
+        output = self.fpn_bottleneck(fpn_outs)
+        return output

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/plugins/__init__.py

@@ -0,0 +1,7 @@
+from .resa_layer import RESA
+from .resa_reducer import RESAReducer
+from .se import SELayer
+from .spatial_conv import SpatialConv
+from .conv_projection_1d import ConvProjection_1D
+from .feature_flip_fusion import FeatureFlipFusion, DCN_v2_Ref
+from .dilated_blocks import predefined_dilated_blocks

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/plugins/conv_projection_1d.py

@@ -0,0 +1,24 @@
+import torch
+import torch.nn as nn
+from torch.nn import functional as F
+
+from ...builder import MODELS
+
+
+@MODELS.register()
+class ConvProjection_1D(torch.nn.Module):
+    # Projection based on line features (1D convs)
+    def __init__(self, num_layers, in_channels, bias=True, k=3):
+        # bias is set as True in FCOS
+        super().__init__()
+        self.num_layers = num_layers
+        self.hidden_layers = nn.ModuleList(nn.Conv1d(in_channels if i > 0 else in_channels,
+                                                     in_channels, kernel_size=k, bias=bias, padding=(k - 1) // 2)
+                                           for i in range(num_layers))
+        self.hidden_norms = nn.ModuleList(nn.BatchNorm1d(in_channels) for _ in range(num_layers))
+
+    def forward(self, x):
+        for conv, norm in zip(self.hidden_layers, self.hidden_norms):
+            x = F.relu(norm(conv(x)))
+
+        return x

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/plugins/dilated_blocks.py

@@ -0,0 +1,18 @@
+import torch.nn as nn
+
+from ...builder import MODELS
+
+
+@MODELS.register()
+def predefined_dilated_blocks(in_channels, mid_channels, dilations):
+    # As in YOLOF
+    blocks = [MODELS.from_dict(
+        dict(
+            name='DilatedBottleneck',
+            in_channels=in_channels,
+            mid_channels=mid_channels,
+            dilation=d
+        )
+    ) for d in dilations]
+
+    return nn.Sequential(*blocks)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/plugins/feature_flip_fusion.py

@@ -0,0 +1,78 @@
+import torch
+import torch.nn as nn
+from mmcv.ops import ModulatedDeformConv2d, modulated_deform_conv2d  # DCNv2
+from torch.nn import functional as F
+from ...builder import MODELS
+
+
+@MODELS.register()
+class DCN_v2_Ref(ModulatedDeformConv2d):
+    """A Encapsulation that acts as normal Conv
+    layers. Modified from mmcv's DCNv2.
+
+    Args:
+        in_channels (int): Same as nn.Conv2d.
+        out_channels (int): Same as nn.Conv2d.
+        kernel_size (int or tuple[int]): Same as nn.Conv2d.
+        stride (int): Same as nn.Conv2d, while tuple is not supported.
+        padding (int): Same as nn.Conv2d, while tuple is not supported.
+        dilation (int): Same as nn.Conv2d, while tuple is not supported.
+        groups (int): Same as nn.Conv2d.
+        bias (bool or str): If specified as `auto`, it will be decided by the
+            norm_cfg. Bias will be set as True if norm_cfg is None, otherwise
+            False.
+    """
+
+    _version = 2
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.conv_offset = nn.Conv2d(
+            self.in_channels * 2,
+            self.deform_groups * 3 * self.kernel_size[0] * self.kernel_size[1],
+            kernel_size=self.kernel_size,
+            stride=self.stride,
+            padding=self.padding,
+            dilation=self.dilation,
+            bias=True)
+        self.init_weights()
+
+    def init_weights(self):
+        super().init_weights()
+        if hasattr(self, 'conv_offset'):
+            self.conv_offset.weight.data.zero_()
+            self.conv_offset.bias.data.zero_()
+
+    def forward(self, x, ref):
+        concat = torch.cat([x, ref], dim=1)
+        out = self.conv_offset(concat)
+        o1, o2, mask = torch.chunk(out, 3, dim=1)
+        offset = torch.cat((o1, o2), dim=1)
+        mask = torch.sigmoid(mask)
+
+        return modulated_deform_conv2d(x, offset, mask, self.weight, self.bias,
+                                       self.stride, self.padding,
+                                       self.dilation, self.groups,
+                                       self.deform_groups)
+
+
+@MODELS.register()
+class FeatureFlipFusion(nn.Module):
+    def __init__(self, channels):
+        super().__init__()
+        self.proj1 = nn.Sequential(
+            nn.Conv2d(channels, channels, kernel_size=1, padding=0),
+            nn.BatchNorm2d(channels)
+        )
+        self.proj2_conv = DCN_v2_Ref(channels, channels, kernel_size=(3, 3), padding=1)
+        self.proj2_norm = nn.BatchNorm2d(channels)
+
+    def forward(self, feature):
+        # B x C x H x W
+        flipped = feature.flip(-1)  # An auto-copy
+
+        feature = self.proj1(feature)
+        flipped = self.proj2_conv(flipped, feature)
+        flipped = self.proj2_norm(flipped)
+
+        return F.relu(feature + flipped)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/plugins/ppm.py

@@ -0,0 +1,34 @@
+import torch.nn as nn
+from torch.nn import functional as F
+
+
+class PPM(nn.ModuleList):
+    """
+    Pooling pyramid module used in PSPNet
+    Args:
+        pool_scales(tuple(int)): Pooling scales used in pooling Pyramid Module
+        applied on the last feature. default: (1, 2, 3, 6)
+    """
+
+    def __init__(self, pool_scales, in_channels, channels, align_corners=False):
+        super(PPM, self).__init__()
+        self.pool_scales = pool_scales
+        self.align_corners = align_corners
+        self.in_channels = in_channels
+        self.channels = channels
+        for pool_scale in pool_scales:
+            self.append(
+                nn.Sequential(
+                    nn.AdaptiveAvgPool2d(pool_scale),
+                    nn.Conv2d(self.in_channels, self.channels, 1),
+                    nn.BatchNorm2d(self.channels),
+                    nn.ReLU()))
+
+    def forward(self, x):
+        ppm_outs = []
+        for ppm in self:
+            ppm_out = ppm(x)
+            upsampled_ppm_out = F.interpolate(ppm_out, size=x.size()[2:], mode='bilinear',
+                                              align_corners=self.align_corners)
+            ppm_outs.append(upsampled_ppm_out)
+        return ppm_outs

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/plugins/resa_layer.py

@@ -0,0 +1,93 @@
+import math
+import torch
+from torch import nn as nn
+from torch.nn import functional as F
+
+from ...builder import MODELS
+from ..._utils import is_tracing
+
+
+@MODELS.register()
+class RESA(nn.Module):
+    # REcurrent Feature-Shift Aggregator in RESA paper
+
+    def __init__(self, num_channels=128, iteration=5, alpha=2.0, trace_arg=None, os=8):
+        super(RESA, self).__init__()
+        # Different from SCNN, RESA uses bias=False & different convolution layers for each stride,
+        # i.e. 4 * iteration layers vs. 4 layers in SCNN, maybe special init is not needed anymore:
+        # https://github.com/ZJULearning/resa/blob/14b0fea6a1ab4f45d8f9f22fb110c1b3e53cf12e/models/resa.py#L21
+        self.iteration = iteration
+        self.alpha = alpha
+        self.conv_d = nn.ModuleList(nn.Conv2d(num_channels, num_channels, (1, 9), padding=(0, 4), bias=False)
+                                    for _ in range(iteration))
+        self.conv_u = nn.ModuleList(nn.Conv2d(num_channels, num_channels, (1, 9), padding=(0, 4), bias=False)
+                                    for _ in range(iteration))
+        self.conv_r = nn.ModuleList(nn.Conv2d(num_channels, num_channels, (9, 1), padding=(4, 0), bias=False)
+                                    for _ in range(iteration))
+        self.conv_l = nn.ModuleList(nn.Conv2d(num_channels, num_channels, (9, 1), padding=(4, 0), bias=False)
+                                    for _ in range(iteration))
+        self._adjust_initializations(num_channels=num_channels)
+        if trace_arg is not None:  # Pre-compute offsets for a TensorRT supported implementation
+            h = (trace_arg['h'] - 1) // os + 1
+            w = (trace_arg['w'] - 1) // os + 1
+            self.offset_h = []
+            self.offset_w = []
+            for i in range(self.iteration):
+                self.offset_h.append(h // 2 ** (self.iteration - i))
+                self.offset_w.append(w // 2 ** (self.iteration - i))
+
+    def _adjust_initializations(self, num_channels=128):
+        # https://github.com/XingangPan/SCNN/issues/82
+        bound = math.sqrt(2.0 / (num_channels * 9 * 5))
+        for i in self.conv_d:
+            nn.init.uniform_(i.weight, -bound, bound)
+        for i in self.conv_u:
+            nn.init.uniform_(i.weight, -bound, bound)
+        for i in self.conv_r:
+            nn.init.uniform_(i.weight, -bound, bound)
+        for i in self.conv_l:
+            nn.init.uniform_(i.weight, -bound, bound)
+
+    def forward(self, x):
+        y = x
+        h, w = y.shape[-2:]
+        if 2 ** self.iteration > min(h, w):
+            print('Too many iterations for RESA, your image size may be too small.')
+
+        # We do indexing here to avoid extra input parameters at __init__(), with almost none computation overhead.
+        # Also, now it won't block arbitrary shaped input.
+        # However, we still need an alternative to Gather for TensorRT
+        # Down
+        for i in range(self.iteration):
+            if is_tracing():
+                temp = torch.cat([y[:, :, self.offset_h[i]:, :], y[:, :, :self.offset_h[i], :]], dim=-2)
+                y = y.add(self.alpha * F.relu(self.conv_d[i](temp)))
+            else:
+                idx = (torch.arange(h) + h // 2 ** (self.iteration - i)) % h
+                y = y + (self.alpha * F.relu(self.conv_d[i](y[:, :, idx, :])))
+        # Up
+        for i in range(self.iteration):
+            if is_tracing():
+                temp = torch.cat([y[:, :, (h - self.offset_h[i]):, :], y[:, :, :(h - self.offset_h[i]), :]], dim=-2)
+                y = y.add(self.alpha * F.relu(self.conv_u[i](temp)))
+            else:
+                idx = (torch.arange(h) - h // 2 ** (self.iteration - i)) % h
+                y = y + (self.alpha * F.relu(self.conv_u[i](y[:, :, idx, :])))
+        # Right
+        for i in range(self.iteration):
+            if is_tracing():
+                temp = torch.cat([y[:, :, :, self.offset_w[i]:], y[:, :, :, :self.offset_w[i]]], dim=-1)
+                y = y.add(self.alpha * F.relu(self.conv_r[i](temp)))
+            else:
+                idx = (torch.arange(w) + w // 2 ** (self.iteration - i)) % w
+                y = y + (self.alpha * F.relu(self.conv_r[i](y[:, :, :, idx])))
+        # Left
+        for i in range(self.iteration):
+            if is_tracing():
+                temp = torch.cat([y[:, :, :, (w - self.offset_w[i]):], y[:, :, :, :(w - self.offset_w[i])]], dim=-1)
+                y = y.add(self.alpha * F.relu(self.conv_l[i](temp)))
+            else:
+                idx = (torch.arange(w) - w // 2 ** (self.iteration - i)) % w
+                y = y + (self.alpha * F.relu(self.conv_l[i](y[:, :, :, idx])))
+
+        return y

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/plugins/resa_reducer.py

@@ -0,0 +1,24 @@
+from torch import nn as nn
+from torch.nn import functional as F
+
+from ...builder import MODELS
+
+
+@MODELS.register()
+class RESAReducer(nn.Module):
+    # Reduce channel (typically to 128), RESA code use no BN nor ReLU
+
+    def __init__(self, in_channels=512, reduce=128, bn_relu=True):
+        super(RESAReducer, self).__init__()
+        self.bn_relu = bn_relu
+        self.conv1 = nn.Conv2d(in_channels, reduce, 1, bias=False)
+        if self.bn_relu:
+            self.bn1 = nn.BatchNorm2d(reduce)
+
+    def forward(self, x):
+        x = self.conv1(x)
+        if self.bn_relu:
+            x = self.bn1(x)
+            x = F.relu(x)
+
+        return x

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/plugins/se.py

@@ -0,0 +1,32 @@
+from torch import nn as nn
+
+from ..._utils import make_divisible
+
+
+class SELayer(nn.Module):
+    """Squeeze-and-Excitation Module.
+    Args:
+        channels (int): The input (and output) channels of the SE layer.
+        ratio (int): Squeeze ratio in SELayer, the intermediate channel will be
+            ``int(channels/ratio)``. Default: 16.
+    """
+
+    def __init__(self, channels, ratio=16, act=nn.ReLU, scale_act=nn.Sigmoid):
+        super(SELayer, self).__init__()
+
+        self.avgpool = nn.AdaptiveAvgPool2d(1)
+        self.fc1 = nn.Conv2d(in_channels=channels, out_channels=make_divisible(channels // ratio, 8), kernel_size=1,
+                             stride=1)
+        self.fc2 = nn.Conv2d(in_channels=make_divisible(channels // ratio, 8), out_channels=channels, kernel_size=1,
+                             stride=1)
+        self.activation = act()
+        self.scale_activation = scale_act()
+
+    def forward(self, x):
+        out = self.avgpool(x)
+        out = self.fc1(out)
+        out = self.activation(out)
+        out = self.fc2(out)
+        out = self.scale_activation(out)
+
+        return x * out

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/plugins/spatial_conv.py

@@ -0,0 +1,96 @@
+import math
+import torch
+from torch import nn as nn
+from torch.nn import functional as F
+
+from ...builder import MODELS
+from ..._utils import is_tracing
+
+
+@MODELS.register()
+class SpatialConv(nn.Module):
+    # SCNN
+
+    def __init__(self, num_channels=128):
+        super().__init__()
+        self.conv_d = nn.Conv2d(num_channels, num_channels, (1, 9), padding=(0, 4))
+        self.conv_u = nn.Conv2d(num_channels, num_channels, (1, 9), padding=(0, 4))
+        self.conv_r = nn.Conv2d(num_channels, num_channels, (9, 1), padding=(4, 0))
+        self.conv_l = nn.Conv2d(num_channels, num_channels, (9, 1), padding=(4, 0))
+        self._adjust_initializations(num_channels=num_channels)
+
+    def _adjust_initializations(self, num_channels=128):
+        # https://github.com/XingangPan/SCNN/issues/82
+        bound = math.sqrt(2.0 / (num_channels * 9 * 5))
+        nn.init.uniform_(self.conv_d.weight, -bound, bound)
+        nn.init.uniform_(self.conv_u.weight, -bound, bound)
+        nn.init.uniform_(self.conv_r.weight, -bound, bound)
+        nn.init.uniform_(self.conv_l.weight, -bound, bound)
+
+    def build_slice(self, vertical, reverse, shape):
+        # refer to https://github.com/harryhan618/SCNN_Pytorch/blob/master/model.py
+        slices = []
+        if vertical: # h
+            concat_dim = 2 
+            length = shape[concat_dim]
+            for i in range(length):
+                slices.append(
+                    (slice(None), slice(None), slice(i, i+1), slice(None))
+                )
+        else: # w
+            concat_dim = 3
+            length = shape[concat_dim]
+            for i in range(length):
+                slices.append(
+                    (slice(None), slice(None), slice(None), slice(i, i+1))
+                )
+        if reverse:
+            slices = slices[::-1]
+        return slices, concat_dim
+
+    def non_inplace_forward(self, input):
+        output = input
+        vertical = [True, True, False, False]
+        reverse = [False, True, False, True]
+        convs = [self.conv_d, self.conv_u, self.conv_r, self.conv_l]
+        for ver, rev, conv in zip(vertical, reverse, convs):
+            slices, dim = self.build_slice(ver, rev, input.shape)
+            output_slices = []
+            for idx, s in enumerate(slices):
+                # the condition is to align with the original forward
+                if (rev is False and idx > 0) or (rev is True and idx < len(slices) - 1 and idx > 0):
+                    output_slices.append(
+                        output[s]  + F.relu(conv(output_slices[-1]))
+                    )
+                else:
+                    output_slices.append(output[s])
+            if rev:
+                output_slices = output_slices[::-1]
+            output = torch.cat(output_slices, dim=dim)
+            if ver is False and rev is True:
+                break
+        return output
+
+    def forward(self, input):
+        output = input
+
+        if is_tracing():
+            # PyTorch index+add_ will be ignored in traced graph
+            # Down
+            for i in range(1, output.shape[2]):
+                output[:, :, i:i + 1, :] = output[:, :, i:i + 1, :].add(F.relu(self.conv_d(output[:, :, i - 1:i, :])))
+            # Up
+            for i in range(output.shape[2] - 2, 0, -1):
+                output[:, :, i:i + 1, :] = output[:, :, i:i + 1, :].add(
+                    F.relu(self.conv_u(output[:, :, i + 1:i + 2, :])))
+            # Right
+            for i in range(1, output.shape[3]):
+                output[:, :, :, i:i + 1] = output[:, :, :, i:i + 1].add(F.relu(self.conv_r(output[:, :, :, i - 1:i])))
+            # Left
+            for i in range(output.shape[3] - 2, 0, -1):
+                output[:, :, :, i:i + 1] = output[:, :, :, i:i + 1].add(
+                    F.relu(self.conv_l(output[:, :, :, i + 1:i + 2])))
+        else:
+            output = self.non_inplace_forward(output)
+
+        return output

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/erfnet_encoder.py

@@ -0,0 +1,70 @@
+# modified from utils/models/segmentation/erfnet.py
+# load pretrained weights during initialization of encoder
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from .common_models import non_bottleneck_1d
+from .builder import MODELS
+
+
+class DownsamplerBlock(nn.Module):
+    def __init__(self, ninput, noutput):
+        super().__init__()
+        self.conv = nn.Conv2d(ninput, noutput-ninput, (3, 3), stride=2, padding=1, bias=True)
+        self.pool = nn.MaxPool2d(2, stride=2)
+        self.bn = nn.BatchNorm2d(noutput, eps=1e-3)
+
+    def forward(self, input):
+        output = torch.cat([self.conv(input), self.pool(input)], 1)
+        output = self.bn(output)
+        return F.relu(output)
+
+@MODELS.register()
+class ERFNetEncoder(nn.Module):
+    def __init__(self, num_classes, dropout_1=0.03, dropout_2=0.3, pretrained_weights=None):
+        super().__init__()
+        self.initial_block = DownsamplerBlock(3, 16)
+        self.layers = nn.ModuleList()
+        self.layers.append(DownsamplerBlock(16, 64))
+        for x in range(0, 5):  # 5 times
+            self.layers.append(non_bottleneck_1d(64, dropout_1, 1))
+        self.layers.append(DownsamplerBlock(64, 128))
+        for x in range(0, 2):  # 2 times
+            self.layers.append(non_bottleneck_1d(128, dropout_2, 2))
+            self.layers.append(non_bottleneck_1d(128, dropout_2, 4))
+            self.layers.append(non_bottleneck_1d(128, dropout_2, 8))
+            self.layers.append(non_bottleneck_1d(128, dropout_2, 16))
+
+        # need to initialize the weights
+        if pretrained_weights is not None:
+            self._load_encoder_weights(pretrained_weights) # Load ImageNet pre-trained weights
+        else:
+            self._init_weights() # initialize random weights
+
+    def _init_weights(self):
+        pass
+
+    def _load_encoder_weights(self, pretrained_weights):
+        # load weights from given file path
+        try:
+            saved_weights = torch.load(pretrained_weights)['state_dict']
+        except FileNotFoundError:
+            raise FileNotFoundError('pretrained_weights is not there! '
+                                    'Please set pretrained_weights=None if you are only testing.')
+        original_weights = self.state_dict()
+        for key in saved_weights.keys():
+            my_key = key.replace('module.features.', '')
+            if my_key in original_weights.keys():
+                original_weights[my_key] = saved_weights[key]
+        self.load_state_dict(original_weights)
+
+    def forward(self, input):
+        output = self.initial_block(input)
+        for layer in self.layers:
+            output = layer(output)
+
+        return output
+
+

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/lane_detection/__init__.py

@@ -0,0 +1,5 @@
+from .lstr import LSTR, cubic_curve_with_projection
+from .resa import RESA_Net
+from .bezier_lane_net import BezierLaneNet
+from . import utils
+from .laneatt import LaneAtt

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/lane_detection/bezier_base.py

@@ -0,0 +1,95 @@
+import torch
+import numpy as np
+
+from .utils import lane_pruning
+from ...curve_utils import BezierCurve
+
+
+class BezierBaseNet(torch.nn.Module):
+    def __init__(self, thresh=0.5, local_maximum_window_size=9):
+        super().__init__()
+        self.thresh = thresh
+        self.local_maximum_window_size = local_maximum_window_size
+
+    def forward(self, *args, **kwargs):
+        raise NotImplementedError
+
+    @staticmethod
+    def bezier_to_coordinates(control_points, existence, resize_shape, dataset, bezier_curve, ppl=56, gap=10):
+        # control_points: L x N x 2
+        H, W = resize_shape
+        cps_of_lanes = []
+        for flag, cp in zip(existence, control_points):
+            if flag:
+                cps_of_lanes.append(cp.tolist())
+        coordinates = []
+        for cps_of_lane in cps_of_lanes:
+            bezier_curve.assign_control_points(cps_of_lane)
+            if dataset == 'tusimple':
+                # Find x for TuSimple's fixed y eval positions (suboptimal)
+                bezier_threshold = 5.0 / H
+                h_samples = np.array([1.0 - (ppl - i) * gap / H for i in range(ppl)], dtype=np.float32)
+                sampled_points = bezier_curve.quick_sample_point(image_size=None)
+                temp = []
+                dis = np.abs(np.expand_dims(h_samples, -1) - sampled_points[:, 1])
+                idx = np.argmin(dis, axis=-1)
+                for i in range(ppl):
+                    h = H - (ppl - i) * gap
+                    if dis[i][idx[i]] > bezier_threshold or sampled_points[idx[i]][0] > 1 or sampled_points[idx[i]][0] < 0:
+                        temp.append([-2, h])
+                    else:
+                        temp.append([sampled_points[idx[i]][0] * W, h])
+                coordinates.append(temp)
+            elif dataset in ['culane', 'llamas']:
+                temp = bezier_curve.quick_sample_point(image_size=None)
+                temp[:, 0] = temp[:, 0] * W
+                temp[:, 1] = temp[:, 1] * H
+                coordinates.append(temp.tolist())
+            else:
+                raise ValueError
+
+        return coordinates
+
+    @torch.no_grad()
+    def inference(self, inputs, input_sizes, gap, ppl, dataset, max_lane=0, forward=True, return_cps=False, n=50):
+        outputs = self.forward(inputs) if forward else inputs  # Support no forwarding inside this function
+        existence_conf = outputs['logits'].sigmoid()
+        existence = existence_conf > self.thresh
+
+        # Test local maxima
+        if self.local_maximum_window_size > 0:
+            _, max_indices = torch.nn.functional.max_pool1d(existence_conf.unsqueeze(1),
+                                                            kernel_size=self.local_maximum_window_size, stride=1,
+                                                            padding=(self.local_maximum_window_size - 1) // 2,
+                                                            return_indices=True)
+            max_indices = max_indices.squeeze(1)  # B x Q
+            indices = torch.arange(0, existence_conf.shape[1],
+                                   dtype=existence_conf.dtype,
+                                   device=existence_conf.device).unsqueeze(0).expand_as(max_indices)
+            local_maxima = max_indices == indices
+            existence *= local_maxima
+
+        control_points = outputs['curves']
+        if max_lane != 0:  # Lane max number prior for testing
+            existence, _ = lane_pruning(existence, existence_conf, max_lane=max_lane)
+
+        if return_cps:
+            image_size = torch.tensor([input_sizes[1][1], input_sizes[1][0]],
+                                      dtype=torch.float32, device=control_points.device)
+            cps = control_points * image_size
+            cps = [cps[i][existence[i]].cpu().numpy() for i in range(existence.shape[0])]
+
+        existence = existence.cpu().numpy()
+        control_points = control_points.cpu().numpy()
+        H, _ = input_sizes[1]
+        b = BezierCurve(order=3, num_sample_points=H if dataset == 'tusimple' else n)
+
+        lane_coordinates = []
+        for j in range(existence.shape[0]):
+            lane_coordinates.append(self.bezier_to_coordinates(control_points=control_points[j], existence=existence[j],
+                                                               resize_shape=input_sizes[1], dataset=dataset,
+                                                               bezier_curve=b, gap=gap, ppl=ppl))
+        if return_cps:
+            return cps, lane_coordinates
+        else:
+            return lane_coordinates

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/lane_detection/bezier_lane_net.py

@@ -0,0 +1,74 @@
+import torch
+import torch.nn as nn
+if torch.__version__ >= '1.6.0':
+    from torch.cuda.amp import autocast
+else:
+    from utils.torch_amp_dummy import autocast
+
+from .bezier_base import BezierBaseNet
+from ..builder import MODELS
+
+
+@MODELS.register()
+class BezierLaneNet(BezierBaseNet):
+    # Curve regression network, similar design as simple object detection (e.g. FCOS)
+    def __init__(self,
+                 backbone_cfg,
+                 reducer_cfg,
+                 dilated_blocks_cfg,
+                 feature_fusion_cfg,
+                 head_cfg,
+                 aux_seg_head_cfg,
+                 image_height=360,
+                 num_regression_parameters=8,
+                 thresh=0.5,
+                 local_maximum_window_size=9):
+        super(BezierLaneNet, self).__init__(thresh, local_maximum_window_size)
+        global_stride = 16
+        branch_channels = 256
+
+        self.backbone = MODELS.from_dict(backbone_cfg)
+        self.reducer = MODELS.from_dict(reducer_cfg)
+        self.dilated_blocks = MODELS.from_dict(dilated_blocks_cfg)
+        self.simple_flip_2d = MODELS.from_dict(feature_fusion_cfg)  # Name kept for legacy weights
+        self.aggregator = nn.AvgPool2d(kernel_size=((image_height - 1) // global_stride + 1, 1), stride=1, padding=0)
+        self.regression_head = MODELS.from_dict(head_cfg)  # Name kept for legacy weights
+        self.proj_classification = nn.Conv1d(branch_channels, 1, kernel_size=1, bias=True, padding=0)
+        self.proj_regression = nn.Conv1d(branch_channels, num_regression_parameters,
+                                         kernel_size=1, bias=True, padding=0)
+        self.segmentation_head = MODELS.from_dict(aux_seg_head_cfg)
+
+    def forward(self, x):
+        # Return shape: B x Q, B x Q x N x 2
+        x = self.backbone(x)
+        if isinstance(x, dict):
+            x = x['out']
+
+        if self.reducer is not None:
+            x = self.reducer(x)
+
+        # Segmentation task
+        if self.segmentation_head is not None:
+            segmentations = self.segmentation_head(x)
+        else:
+            segmentations = None
+
+        if self.dilated_blocks is not None:
+            x = self.dilated_blocks(x)
+
+        with autocast(False):  # TODO: Support fp16 like mmcv
+            x = self.simple_flip_2d(x.float())
+        x = self.aggregator(x)[:, :, 0, :]
+
+        x = self.regression_head(x)
+        logits = self.proj_classification(x).squeeze(1)
+        curves = self.proj_regression(x)
+
+        return {'logits': logits,
+                'curves': curves.permute(0, 2, 1).reshape(curves.shape[0], -1, curves.shape[-2] // 2, 2).contiguous(),
+                'segmentations': segmentations}
+
+    def eval(self, profiling=False):
+        super().eval()
+        if profiling:
+            self.segmentation_head = None

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/lane_detection/laneatt.py

@@ -0,0 +1,342 @@
+# Refactored from lucastabelini/LaneATT
+# Diffs:
+# 1. we changed lane rep to 74 numbers (start, len, 72 offsets)
+# 2. we use a cleaner line nms dynamically loaded (input only 74 numbers, not 77)
+# 3. we removed unnecessary inputs & outputs in post-processing funcs
+# 4. we removed B-Spline interpolation post-processing to provide fair comparisons (test 74.88 for resnet18-CULane in official code)
+# After these simplifications & refactors, our testing procedure seems even better than the original
+
+import math
+import torch
+import numpy as np
+import torch.nn as nn
+from torch.nn import functional as F
+from scipy.interpolate import splprep, splev
+from .._utils import is_tracing
+
+try:
+    from ...csrc.apis import line_nms
+    print('Successfully complied line nms for LaneATT.')
+except:
+    from ...common import warnings
+    warnings.warn('Can\'t complie line nms op for LaneATT. Set verbose=True for load in /utils/csrc/apis.py L9 for details.')
+from ..builder import MODELS
+
+
+@MODELS.register()
+class LaneAtt(nn.Module):
+    # Anchor angles, same ones used in Line-CNN
+    left_angles = [72., 60., 49., 39., 30., 22.]
+    right_angles = [108., 120., 131., 141., 150., 158.]
+    bottom_angles = [165., 150., 141., 131., 120., 108., 100., 90., 80., 72., 60., 49., 39., 30., 15.]
+
+    def __init__(self,
+                 backbone_cfg,
+                 backbone_channels,
+                 backbone_os,
+                 num_points=72,
+                 img_w=640,
+                 img_h=360,
+                 topk_anchors=None,
+                 anchor_freq_path=None,
+                 anchor_feat_channels=None,
+                 conf_thres=None,
+                 nms_thres=0,
+                 nms_topk=3000,
+                 trace_arg=None):
+        super().__init__()
+        self.backbone = MODELS.from_dict(backbone_cfg)
+        self.backbone_channels = backbone_channels
+        self.stride = backbone_os
+        self.num_strips = num_points - 1
+        self.num_offsets = num_points
+        self.img_h = img_h
+        self.img_w = img_w
+        self.featmap_h = img_h // self.stride
+        self.featmap_w = img_w // self.stride
+        self.anchor_ys = torch.linspace(1, 0, steps=self.num_offsets, dtype=torch.float32)
+        self.anchor_cut_ys = torch.linspace(1, 0, steps=self.featmap_h, dtype=torch.float32)
+        self.anchor_feat_channels = anchor_feat_channels
+        # nms config
+        self.conf_thres = conf_thres
+        self.nms_thres = nms_thres
+        self.nms_topk = nms_topk
+
+        if trace_arg is not None:  # Pre-compute
+            attention_matrix = torch.eye(topk_anchors).repeat(trace_arg['bs'], 1, 1)
+            self.pre_non_diag_inds = torch.nonzero(attention_matrix == 0., as_tuple=False)
+
+        # generate anchors
+        self.anchors, self.anchors_cut = self.generate_anchors(lateral_n=72, bottom_n=128)
+        # Filter masks if `anchors_freq_path` is provided
+        if anchor_freq_path is not None:
+            anchors_mask = torch.load(anchor_freq_path).cpu()
+            assert topk_anchors is not None, 'topk_anchors cannot be None'
+            idx = torch.argsort(anchors_mask, descending=True)[: topk_anchors]
+            self.anchors = self.anchors[idx]
+            self.anchors_cut = self.anchors_cut[idx]
+
+        # pre compute indices for the anchor pooling
+        self.cut_zs, self.cut_ys, self.cut_xs, self.invalid_mask = self.compute_anchor_cut_indices(
+            self.anchor_feat_channels, self.featmap_w, self.featmap_h)
+
+        # Setup ans initialize layers
+        self.conv1 = nn.Conv2d(self.backbone_channels, self.anchor_feat_channels, kernel_size=1)
+        self.cls_layer = nn.Linear(2 * self.anchor_feat_channels * self.featmap_h, 2)
+        self.reg_layer = nn.Linear(2 * self.anchor_feat_channels * self.featmap_h, self.num_offsets + 1)
+        self.attention_layer = nn.Linear(self.anchor_feat_channels * self.featmap_h, len(self.anchors) - 1)
+        self.initialize_layer(self.attention_layer)
+        self.initialize_layer(self.conv1)
+        self.initialize_layer(self.cls_layer)
+        self.initialize_layer(self.reg_layer)
+
+    def generate_anchors(self, lateral_n, bottom_n):
+        left_anchors, left_cut = self.generate_side_anchors(self.left_angles, x=0., nb_origins=lateral_n)
+        right_anchors, right_cut = self.generate_side_anchors(self.right_angles, x=1., nb_origins=lateral_n)
+        bottom_anchors, bottom_cut = self.generate_side_anchors(self.bottom_angles, y=1., nb_origins=bottom_n)
+
+        return torch.cat([left_anchors, bottom_anchors, right_anchors]), \
+            torch.cat([left_cut, bottom_cut, right_cut])
+
+    def generate_side_anchors(self, angles, nb_origins, x=None, y=None):
+        if x is None and y is not None:
+            starts = [(x, y) for x in np.linspace(1., 0., num=nb_origins)]
+        elif x is not None and y is None:
+            starts = [(x, y) for y in np.linspace(1., 0., num=nb_origins)]
+        else:
+            raise Exception('Please define exactly one of `x` or `y` (not neither nor both)')
+
+        n_anchors = nb_origins * len(angles)
+
+        # each row, first for x and second for y:
+        # 2 scores, 1 start_y, start_x, 1 length, num_points coordinates
+        # score[0] = negative prob, score[0] = positive prob
+        anchors = torch.zeros((n_anchors, 2 + self.num_offsets))
+        anchors_cut = torch.zeros((n_anchors, 2 + self.featmap_h))
+        for i, start in enumerate(starts):
+            for j, angle in enumerate(angles):
+                k = i * len(angles) + j
+                anchors[k] = self.generate_anchor(start, angle)
+                anchors_cut[k] = self.generate_anchor(start, angle, cut=True)
+
+        return anchors, anchors_cut
+
+    def generate_anchor(self, start, angle, cut=False):
+        if cut:
+            anchor_ys = self.anchor_cut_ys
+            anchor = torch.zeros(2 + self.featmap_h)
+        else:
+            anchor_ys = self.anchor_ys
+            anchor = torch.zeros(2 + self.num_offsets)
+        angle = angle * math.pi / 180.  # degrees to radians
+        start_x, start_y = start
+        anchor[0] = 1 - start_y  # using left bottom as the (0, 0) of the axis ?
+        anchor[1] = start_x
+        anchor[2:] = (start_x + (1 - anchor_ys - 1 + start_y) / math.tan(angle)) * self.img_w
+
+        return anchor
+
+    def compute_anchor_cut_indices(self, num_channels, feat_w, feat_h):
+        # definitions
+        num_proposal = len(self.anchors_cut)
+
+        # indexing
+        # num_anchors x feat_h
+        unclamped_xs = torch.flip((self.anchors_cut[:, 2:] / self.stride).round().long(), dims=(1, ))
+        unclamped_xs = unclamped_xs[..., None]
+        # num_channels x num_anchors x feat_h --> num_channels * num_anchors * feat_h x 1
+        unclamped_xs = torch.repeat_interleave(unclamped_xs, num_channels, dim=0).reshape(-1, 1)
+        cut_xs = torch.clamp(unclamped_xs, 0, feat_w - 1)
+        unclamped_xs = unclamped_xs.reshape(num_proposal, num_channels, feat_h, 1)
+        invalid_mask = (unclamped_xs < 0) | (unclamped_xs > feat_w)
+        cut_ys = torch.arange(0, feat_h)
+        cut_ys = cut_ys.repeat(num_channels * num_proposal)[:, None].reshape(num_proposal, num_channels, feat_h)
+        cut_ys = cut_ys.reshape(-1, 1)
+        cut_zs = torch.arange(num_channels).repeat_interleave(feat_h).repeat(num_proposal)[:, None]
+
+        return cut_zs, cut_ys, cut_xs, invalid_mask
+
+    def cut_anchor_features(self, features):
+        # definitions
+        batch_size = features.shape[0]
+        n_proposals = len(self.anchors)
+        n_fmaps = features.shape[1]
+        batch_anchor_features = torch.zeros((batch_size, n_proposals, n_fmaps, self.featmap_h, 1),
+                                            device=features.device)
+
+        # actual cutting
+        for batch_idx, img_features in enumerate(features):
+            rois = img_features[self.cut_zs, self.cut_ys, self.cut_xs].view(n_proposals, n_fmaps, self.featmap_h, 1)
+            rois[self.invalid_mask] = 0
+            batch_anchor_features[batch_idx] = rois
+
+        return batch_anchor_features
+
+    @staticmethod
+    def initialize_layer(layer):
+        if isinstance(layer, (nn.Conv2d, nn.Linear)):
+            torch.nn.init.normal_(layer.weight, mean=0., std=0.001)
+            if layer.bias is not None:
+                torch.nn.init.constant_(layer.bias, 0)
+
+    def cuda(self, device=None):
+        cuda_self = super().cuda(device)
+        cuda_self.anchors = cuda_self.anchors.cuda(device)
+        cuda_self.anchor_ys = cuda_self.anchor_ys.cuda(device)
+        cuda_self.cut_zs = cuda_self.cut_zs.cuda(device)
+        cuda_self.cut_ys = cuda_self.cut_ys.cuda(device)
+        cuda_self.cut_xs = cuda_self.cut_xs.cuda(device)
+        cuda_self.invalid_mask = cuda_self.invalid_mask.cuda(device)
+        return cuda_self
+
+    def to(self, *args, **kwargs):
+        device_self = super().to(*args, **kwargs)
+        device_self.anchors = device_self.anchors.to(*args, **kwargs)
+        device_self.anchor_ys = device_self.anchor_ys.to(*args, **kwargs)
+        device_self.cut_zs = device_self.cut_zs.to(*args, **kwargs)
+        device_self.cut_ys = device_self.cut_ys.to(*args, **kwargs)
+        device_self.cut_xs = device_self.cut_xs.to(*args, **kwargs)
+        device_self.invalid_mask = device_self.invalid_mask.to(*args, **kwargs)
+        return device_self
+
+    def forward(self, x):
+        batch_features = self.backbone(x)['out']
+
+        batch_features = self.conv1(batch_features)
+        # batchsize x num_proposals x anchor_feat_channels x featmap_h x 1
+        batch_anchor_features = self.cut_anchor_features(batch_features)
+
+        # join proposals from all images into a single proposals features batch
+        batch_anchor_features = batch_anchor_features.view(-1, self.anchor_feat_channels * self.featmap_h)
+        # add attention features
+        softmax = nn.Softmax(dim=1)
+        scores = self.attention_layer(batch_anchor_features)
+        attention = softmax(scores).reshape(x.shape[0], len(self.anchors), -1)
+        attention_matrix = torch.eye(attention.shape[1], device=x.device).repeat(x.shape[0], 1, 1)
+        if is_tracing():
+            # Use pre-computed nonzero results
+            non_diag_inds = self.pre_non_diag_inds.to(attention_matrix.device)
+        else:
+            non_diag_inds = torch.nonzero(attention_matrix == 0., as_tuple=False)
+        attention_matrix[:] = 0
+        attention_matrix[non_diag_inds[:, 0], non_diag_inds[:, 1], non_diag_inds[:, 2]] = attention.flatten()
+        batch_anchor_features = batch_anchor_features.reshape(x.shape[0], len(self.anchors), -1)
+        attention_features = torch.bmm(torch.transpose(batch_anchor_features, 1, 2),
+                                       torch.transpose(attention_matrix, 1, 2)).transpose(1, 2)
+        attention_features = attention_features.reshape(-1, self.anchor_feat_channels * self.featmap_h)
+        batch_anchor_features = batch_anchor_features.reshape(-1, self.anchor_feat_channels * self.featmap_h)
+        batch_anchor_features = torch.cat((attention_features, batch_anchor_features), dim=1)
+
+        # predict
+        cls_logits = self.cls_layer(batch_anchor_features)
+        reg = self.reg_layer(batch_anchor_features)
+
+        # Undo joining
+        cls_logits = cls_logits.reshape(x.shape[0], -1, cls_logits.shape[1])
+        reg = reg.reshape(x.shape[0], -1, reg.shape[1])
+
+        # Add offset to anchors
+        reg_proposals = torch.zeros((*cls_logits.shape[:2], self.num_offsets + 2), device=x.device)
+        reg_proposals += self.anchors
+        reg_proposals[:, :, 1:] += reg
+
+        return {
+            'offsets': reg_proposals[..., 2:],  # B x M x 72
+            'starts': reg_proposals[..., 0],  # B x M x 1
+            'lengths': reg_proposals[..., 1],  # B x M x 1
+            'logits': cls_logits  # B x M x 2
+        }
+
+    @torch.no_grad()
+    def inference(self, inputs, input_sizes, forward=True, *args, **kwargs):
+        outputs = self.forward(inputs) if forward else inputs  # Support no forwarding inside this function
+        to_tusimple = True if args[1] == "tusimple" else False
+        batch_regs = self.nms(outputs)
+        # the number of lanes is 1 ???
+        decoded = []
+        for regs in batch_regs:
+            regs[:, 1] = torch.round(regs[:, 1])  # length
+            if regs.shape[0] == 0:
+                decoded.append([])
+                continue
+            pred = self.proposals_to_pred(regs, input_sizes[1], to_tusimple)
+            decoded.append(pred)
+        return decoded
+
+    @torch.no_grad()
+    def nms(self, batch_proposals):
+        proposals_list = []
+        for i in range(len(batch_proposals['logits'])):
+            scores = F.softmax(batch_proposals['logits'][i], dim=1)[:, 1]
+            regs = torch.cat([batch_proposals['starts'][i][..., None],
+                              batch_proposals['lengths'][i][..., None],
+                              batch_proposals['offsets'][i]], dim=1)
+            if self.conf_thres is not None:
+                # apply confidence threshold
+                above_threshold = scores > self.conf_thres
+                regs = regs[above_threshold]
+                scores = scores[above_threshold]
+            if regs.shape[0] == 0:
+                proposals_list.append(regs[[]])
+                continue
+            keep, num_to_keep, _ = line_nms(regs, scores, self.nms_thres, self.nms_topk)
+            keep = keep[:num_to_keep]
+            regs = regs[keep]
+            proposals_list.append(regs)
+
+        return proposals_list
+
+    def proposals_to_pred(self, proposals, image_size, to_tusimple=False):
+        self.anchor_ys = self.anchor_ys.to(proposals.device)
+        lanes = []
+        for lane in proposals:
+            lane_xs = lane[2:] / self.img_w
+
+            # start end length in 0-72
+            start = int(round(lane[0].item() * self.num_strips))
+            length = int(round(lane[1].item()))
+            end = start + length - 1
+            end = min(end, len(self.anchor_ys) - 1)
+
+            # end = label_end
+            # if the proposal does not start at the bottom of the image,
+            # extend its proposal until the x is outside the image
+            mask = ~((((lane_xs[:start] >= 0.) &
+                       (lane_xs[:start] <= 1.)).cpu().numpy()[::-1].cumprod()[::-1]).astype(np.bool))
+            lane_xs[end + 1:] = -2
+            lane_xs[:start][mask] = -2
+            lane_ys = self.anchor_ys[lane_xs >= 0]
+            lane_xs = lane_xs[lane_xs >= 0]
+            lane_xs = lane_xs.flip(0)
+            lane_ys = lane_ys.flip(0)
+            if len(lane_xs) <= 1:
+                continue
+            points = torch.stack((lane_xs.reshape(-1, 1), lane_ys.reshape(-1, 1)), dim=1).squeeze(2)
+            points = points.cpu().numpy()
+            lane_coords = []
+            for i in range(points.shape[0]):
+                lane_coords.append([points[i, 0] * float(image_size[1]), points[i, 1] * float(image_size[0])])
+            if to_tusimple:
+                lanes.append(self.convert_to_tusimple(lane_coords))
+            else:
+                lanes.append(lane_coords)
+
+        return lanes
+
+    def convert_to_tusimple(self, points, n=200, bezier_threshold=5):
+        """Spline interpolation of a lane. Used on the predictions"""
+        x = [x for x, _ in points]
+        y = [y for _, y in points]
+        tck, _ = splprep([x, y], s=0, t=n, k=min(3, len(points) - 1))
+        u = np.linspace(0., 1., n)
+        rep_points = np.array(splev(u, tck)).T
+        h_samples = [(160 + y * 10) for y in range(56)]
+        temp = []
+        for h_sample in h_samples:
+            dis = np.abs(h_sample - rep_points[:, 1])
+            idx = np.argmin(dis)
+            if dis[idx] > bezier_threshold:
+                temp.append([-2, h_sample])
+            else:
+                temp.append([round(rep_points[:, 0][idx], 3), h_sample])
+        return temp

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/lane_detection/lstr.py

@@ -0,0 +1,176 @@
+# Adapted from liuruijin17/LSTR
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from .utils import lane_pruning
+from ..transformer import build_transformer, build_position_encoding
+from ..mlp import MLP
+from .._utils import is_tracing
+from ..builder import MODELS
+
+
+def cubic_curve_with_projection(coefficients, y):
+    # The cubic curve model from LSTR (considers projection to image plane)
+    # Return x coordinates
+    # coefficients: [d1, d2, ..., 6]
+    # 6 coefficients: [k", f", m", n", b", b''']
+    # y: [d1, d2, ..., N]
+    y = y.permute(-1, *[i for i in range(len(y.shape) - 1)])  # -> [N, d1, d2, ...]
+    x = coefficients[..., 0] / (y - coefficients[..., 1]) ** 2 \
+        + coefficients[..., 2] / (y - coefficients[..., 1]) \
+        + coefficients[..., 3] \
+        + coefficients[..., 4] * y \
+        - coefficients[..., 5]
+
+    return x.permute(*[i + 1 for i in range(len(x.shape) - 1)], 0)  # [d1, d2, ... , N]
+
+
+@MODELS.register()
+class LSTR(nn.Module):
+    def __init__(self,
+                 expansion=1,  # Expansion rate (1x for TuSimple & 2x for CULane)
+                 num_queries=7,  # Maximum number of lanes
+                 aux_loss=True,  # Important for transformer-based methods
+                 pos_type='sine',
+                 drop_out=0.1,
+                 num_heads=2,
+                 enc_layers=2,
+                 dec_layers=2,
+                 pre_norm=False,
+                 return_intermediate=True,
+                 lsp_dim=8,
+                 mlp_layers=3,
+                 backbone_cfg=None,
+                 thresh=0.95,
+                 trace_arg=None):
+        super().__init__()
+
+        self.thresh = thresh
+        self.backbone = MODELS.from_dict(backbone_cfg)
+        hidden_dim = 32 * expansion
+        self.aux_loss = aux_loss
+        self.position_embedding = build_position_encoding(hidden_dim=hidden_dim, position_embedding=pos_type)
+        if trace_arg is not None:  # Pre-compute embeddings
+            trace_arg['h'] = (trace_arg['h'] - 1) // 32 + 1
+            trace_arg['w'] = (trace_arg['w'] - 1) // 32 + 1
+            x = torch.zeros((trace_arg['bs'], trace_arg['h'], trace_arg['w']), dtype=torch.bool)
+            y = torch.zeros((trace_arg['bs'], 128 * expansion, trace_arg['h'], trace_arg['w']), dtype=torch.float32)
+            self.pos = torch.nn.Parameter(data=self.position_embedding(y, x), requires_grad=False)
+        self.query_embed = nn.Embedding(num_queries, hidden_dim)
+        self.input_proj = nn.Conv2d(128 * expansion, hidden_dim, kernel_size=1)  # Same channel as layer4
+
+        self.transformer = build_transformer(hidden_dim=hidden_dim,
+                                             dropout=drop_out,
+                                             nheads=num_heads,
+                                             dim_feedforward=128 * expansion,
+                                             enc_layers=enc_layers,
+                                             dec_layers=dec_layers,
+                                             pre_norm=pre_norm,
+                                             return_intermediate_dec=return_intermediate)
+
+        # Original LSTR: 3 classes + CE (softmax), we use 2
+        self.class_embed = nn.Linear(hidden_dim, 2)
+
+        self.specific_embed = MLP(hidden_dim, hidden_dim, lsp_dim - 4, mlp_layers)  # Specific for each lane
+        self.shared_embed = MLP(hidden_dim, hidden_dim, 4, mlp_layers)  # 4 shared curve coefficients
+
+    def forward(self, images, padding_masks=None):
+        # images: B x C x H x W
+        # padding_masks: B x H x W (0 or 1 -> ignored)
+        p = self.backbone(images)['out']
+
+        # Padding mask (for paddings added in transforms)
+        if is_tracing():
+            pos = self.pos
+        else:
+            if padding_masks is None:  # Make things easier for testing (assume no padding)
+                padding_masks = torch.zeros((p.shape[0], p.shape[2], p.shape[3]), dtype=torch.bool, device=p.device)
+            else:
+                padding_masks = F.interpolate(padding_masks[None].float(), size=p.shape[-2:]).to(torch.bool)[0]
+            pos = self.position_embedding(p, padding_masks)
+
+        hs, _ = self.transformer(self.input_proj(p), padding_masks, self.query_embed.weight, pos)
+        output_class = self.class_embed(hs)
+        output_specific = self.specific_embed(hs)
+        output_shared = self.shared_embed(hs)
+        output_shared = torch.mean(output_shared, dim=-2, keepdim=True)  # Why not take mean on input and simply expand?
+        output_shared = output_shared.repeat(1, 1, output_specific.shape[2], 1)
+
+        # Keep this for consistency with official LSTR: [upper, lower, k", f", m", n", b", b''']
+        output_curve = torch.cat([output_specific[:, :, :, :2],
+                                  output_shared, output_specific[:, :, :, 2:]], dim=-1)
+
+        out = {'logits': output_class[-1], 'curves': output_curve[-1]}  # Last layer result
+        if self.aux_loss:
+            out['aux'] = self._set_aux_loss(output_class, output_curve)  # All intermediate results
+
+        return out
+
+    def eval(self, profiling=False):
+        super().eval()
+        if profiling:
+            self.aux_loss = False
+            self.transformer.decoder.return_intermediate = False
+
+    @torch.no_grad()
+    def inference(self, inputs, input_sizes, gap, ppl, dataset, max_lane=0, forward=True, **kwargs):
+        outputs = self.forward(inputs) if forward else inputs  # Support no forwarding inside this function
+        existence_conf = outputs['logits'].softmax(dim=-1)[..., 1]
+        existence = existence_conf > self.thresh
+        if max_lane != 0:  # Lane max number prior for testing
+            existence, _ = lane_pruning(existence, existence_conf, max_lane=max_lane)
+
+        existence = existence.cpu().numpy()
+        # Get coordinates for lanes
+        lane_coordinates = []
+        for j in range(existence.shape[0]):
+            lane_coordinates.append(self.coefficients_to_coordinates(outputs['curves'][j, :, 2:], existence[j],
+                                    resize_shape=input_sizes[1], dataset=dataset, ppl=ppl,
+                                    gap=gap, curve_function=cubic_curve_with_projection,
+                                    upper_bound=outputs['curves'][j, :, 0],
+                                    lower_bound=outputs['curves'][j, :, 1]))
+
+        return lane_coordinates
+
+    @staticmethod
+    def coefficients_to_coordinates(coefficients, existence, resize_shape, dataset, ppl, gap, curve_function,
+                                    upper_bound, lower_bound):
+        # For methods that predict coefficients of polynomials,
+        # works with normalized coordinates (in range 0.0 ~ 1.0).
+        # Restricted to single image to align with other methods' codes
+        H, W = resize_shape
+        if dataset == 'tusimple':  # Annotation start at 10 pixel away from bottom
+            y = torch.tensor([1.0 - (ppl - i) * gap / H for i in range(ppl)],
+                             dtype=coefficients.dtype, device=coefficients.device)
+        elif dataset in ['culane', 'llamas']:  # Annotation start at bottom
+            y = torch.tensor([1.0 - i * gap / H for i in range(ppl)],
+                             dtype=coefficients.dtype, device=coefficients.device)
+        else:
+            raise ValueError
+        coords = curve_function(coefficients=coefficients, y=y.unsqueeze(0).expand(coefficients.shape[0], -1))
+
+        # Delete outside points according to predicted upper & lower boundaries
+        coordinates = []
+        for i in range(existence.shape[0]):
+            if existence[i]:
+                # Note that in image coordinate system, (0, 0) is the top-left corner
+                valid_points = (coords[i] >= 0) * (coords[i] <= 1) * (y < lower_bound[i]) * (y > upper_bound[i])
+                if valid_points.sum() < 2:  # Same post-processing technique as segmentation methods
+                    continue
+                if dataset == 'tusimple':  # Invalid sample points need to be included as negative value, e.g. -2
+                    coordinates.append([[(coords[i][j] * W).item(), H - (ppl - j) * gap]
+                                        if valid_points[j] else [-2, H - (ppl - j) * gap] for j in range(ppl)])
+                elif dataset in ['culane', 'llamas']:
+                    coordinates.append([[(coords[i][j] * W).item(), H - j * gap]
+                                        for j in range(ppl) if valid_points[j]])
+                else:
+                    raise ValueError
+
+        return coordinates
+
+    @torch.jit.unused
+    def _set_aux_loss(self, output_class, output_curve):
+        # this is a workaround to make torchscript happy, as torchscript
+        # doesn't support dictionary with non-homogeneous values, such
+        # as a dict having both a Tensor and a list.
+        return [{'logits': a, 'curves': b} for a, b in zip(output_class[:-1], output_curve[:-1])]

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/lane_detection/resa.py

@@ -0,0 +1,50 @@
+# Adapted from ZJULearning/resa
+# Better to use a decoupled implementation,
+# costs more codes, but clear.
+# Diff from RESA official code:
+# 1. we use BN+ReLU in channel reducer
+# 2. we always use the BUSD decoder in the paper (official code does not use BUSD in CULane)
+# 3. we always use 5 RESA iterations (4 in official code)
+# 4. we use a higher capacity lane existence classifier (same as ERFNet/ENet baseline)
+# 5. we use the SCNN sqrt(5) init trick for RESA, which
+# 5.1. enables fewer warmup steps
+# 5.2. combined with 4, produces slightly better performance
+# 6. we do not use horizontal flip or cutting height in loading, in which
+# 6.1. flip does not help performance (at least on the val set)
+# 6.2. w.o. cutting height trick probably is the main reason for our lower performance, but we can't use it since
+# other pytorch-auto-drive models do not use it.
+import torch.nn as nn
+
+from ..builder import MODELS
+
+
+@MODELS.register()
+class RESA_Net(nn.Module):
+    def __init__(self,
+                 backbone_cfg,
+                 reducer_cfg,
+                 spatial_conv_cfg,
+                 classifier_cfg,
+                 lane_classifier_cfg,
+                 trace_arg=None):
+        super().__init__()
+        self.backbone = MODELS.from_dict(backbone_cfg)
+        # self.channel_reducer = RESAReducer(in_channels=in_channels, reduce=channel_reduce, bn_relu=False)
+        self.channel_reducer = MODELS.from_dict(reducer_cfg)
+        self.spatial_conv = MODELS.from_dict(spatial_conv_cfg, trace_arg=trace_arg)
+        self.decoder = MODELS.from_dict(classifier_cfg)
+        # self.decoder = PlainDecoder(num_classes=num_classes)
+        self.lane_classifier = MODELS.from_dict(lane_classifier_cfg)
+        # self.lane_classifier = RESALaneExist(num_output=num_classes - 1, flattened_size=flattened_size)
+
+    def forward(self, x):
+        x = self.backbone(x)
+        if isinstance(x, dict):
+            x = x['out']
+        x = self.channel_reducer(x)
+        x = self.spatial_conv(x)
+
+        res = {'out': self.decoder(x),
+               'lane': self.lane_classifier(x)}
+
+        return res

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/lane_detection/utils.py

@@ -0,0 +1,14 @@
+# Setup to temporarily avoid recursive imports
+
+
+def lane_pruning(existence, existence_conf, max_lane):
+    # Prune lanes based on confidence (a max number constrain for lanes in an image)
+    # Maybe too slow (but should be faster than topk/sort),
+    # consider batch size >> max number of lanes
+    while (existence.sum(dim=1) > max_lane).sum() > 0:
+        indices = (existence.sum(dim=1, keepdim=True) > max_lane).expand_as(existence) * \
+                  (existence_conf == existence_conf.min(dim=1, keepdim=True).values)
+        existence[indices] = 0
+        existence_conf[indices] = 1.1  # So we can keep using min
+
+    return existence, existence_conf

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/mlp.py

@@ -0,0 +1,22 @@
+from torch import nn
+import torch.nn.functional as F
+
+from .builder import MODELS
+
+
+@MODELS.register()
+class MLP(nn.Module):
+    # Modified from facebookresearch/detr
+    # Very simple multi-layer perceptron (also called FFN)
+
+    def __init__(self, input_dim, hidden_dim, output_dim, num_layers):
+        super().__init__()
+        self.num_layers = num_layers
+        h = [hidden_dim] * (num_layers - 1)
+        self.layers = nn.ModuleList(nn.Linear(n, k) for n, k in zip([input_dim] + h, h + [output_dim]))
+
+    def forward(self, x):
+        for i, layer in enumerate(self.layers):
+            x = F.relu(layer(x)) if i < self.num_layers - 1 else layer(x)
+
+        return x

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/mobilenet_v2.py

@@ -0,0 +1,156 @@
+# Modified from mmsegmentation code, referenced from torchvision
+
+import torch.nn as nn
+from .builder import MODELS
+from ._utils import make_divisible
+from .common_models import InvertedResidual
+from .utils import load_state_dict_from_url
+
+
+@MODELS.register()
+class MobileNetV2Encoder(nn.Module):
+    """MobileNetV2 backbone (up to second-to-last feature map).
+    This backbone is the implementation of
+    `MobileNetV2: Inverted Residuals and Linear Bottlenecks
+    <https://arxiv.org/abs/1801.04381>`_.
+    Args:
+        widen_factor (float): Width multiplier, multiply number of
+            channels in each layer by this amount. Default: 1.0.
+        strides (Sequence[int], optional): Strides of the first block of each
+            layer. If not specified, default config in ``arch_setting`` will
+            be used.
+        dilations (Sequence[int]): Dilation of each layer.
+        out_indices (None or Sequence[int]): Output from which stages.
+            Default: (7, ).
+        frozen_stages (int): Stages to be frozen (all param fixed).
+            Default: -1, which means not freezing any parameters.
+        norm_eval (bool): Whether to set norm layers to eval mode, namely,
+            freeze running stats (mean and var). Note: Effect on Batch Norm
+            and its variants only. Default: False.
+        pretrained (str, optional): model pretrained path. Default: None
+        out_stride (int): the output stride of the output feature map
+    """
+    # Parameters to build layers. 3 parameters are needed to construct a
+    # layer, from left to right: expand_ratio, channel, num_blocks.
+    arch_settings = [[1, 16, 1], [6, 24, 2], [6, 32, 3], [6, 64, 4],
+                     [6, 96, 3], [6, 160, 3], [6, 320, 1]]
+
+    def __init__(self, widen_factor=1., strides=(1, 2, 2, 2, 1, 2, 1), dilations=(1, 1, 1, 1, 1, 1, 1),
+                 out_indices=(1, 2, 4, 6), frozen_stages=-1, norm_eval=False, pretrained=None,
+                 progress=True, out_stride=0):
+        super(MobileNetV2Encoder, self).__init__()
+        self.pretrained = pretrained
+        self.widen_factor = widen_factor
+        self.strides = strides
+        self.dilations = dilations
+        assert len(strides) == len(dilations) == len(self.arch_settings)
+        self.out_indices = out_indices
+        for index in out_indices:
+            if index not in range(0, 7):
+                raise ValueError('the item in out_indices must in range(0, 7). But received {index}')
+        if frozen_stages not in range(-1, 7):
+            raise ValueError('frozen_stages must be in range(-1, 7). But received {frozen_stages}')
+        self.out_indices = out_indices
+        self.frozen_stages = frozen_stages
+        self.norm_eval = norm_eval
+        self.out_stride = out_stride
+        self.in_channels = make_divisible(32 * widen_factor, 8)
+        self.conv1 = nn.Sequential(
+            nn.Conv2d(in_channels=3, out_channels=self.in_channels, kernel_size=3, stride=2, padding=1, bias=False),
+            nn.BatchNorm2d(self.in_channels),
+            nn.ReLU6()
+        )
+        self.layers = []
+        for i, layer_cfg in enumerate(self.arch_settings):
+            expand_ratio, channel, num_blocks = layer_cfg
+            stride = self.strides[i]
+            dilation = self.dilations[i]
+            out_channels = make_divisible(channel * widen_factor, 8)
+            inverted_res_layer = self.make_layer(
+                out_channels=out_channels,
+                num_blocks=num_blocks,
+                stride=stride,
+                dilation=dilation,
+                expand_ratio=expand_ratio)
+            layer_name = f'layer{i + 1}'
+            self.add_module(layer_name, inverted_res_layer)
+            self.layers.append(layer_name)
+
+        if self.pretrained is None:
+            self.weight_initialization()
+        else:
+            self.load_pretrained(progress=progress)
+
+    def load_pretrained(self, progress):
+        state_dict = load_state_dict_from_url(self.pretrained, progress=progress)
+        self_state_dict = self.state_dict()
+        self_keys = list(self_state_dict.keys())
+        for i, (_, v) in enumerate(state_dict.items()):
+            if i > len(self_keys) - 1:
+                break
+            self_state_dict[self_keys[i]] = v
+        self.load_state_dict(self_state_dict)
+
+    def weight_initialization(self):
+        # weight initialization
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                nn.init.kaiming_normal_(m.weight, mode="fan_out")
+                if m.bias is not None:
+                    nn.init.zeros_(m.bias)
+            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):
+                nn.init.ones_(m.weight)
+                nn.init.zeros_(m.bias)
+            elif isinstance(m, nn.Linear):
+                nn.init.normal_(m.weight, 0, 0.01)
+                nn.init.zeros_(m.bias)
+
+    def make_layer(self, out_channels, num_blocks, stride, dilation,
+                   expand_ratio):
+        """Stack InvertedResidual blocks to build a layer for MobileNetV2.
+        Args:
+            out_channels (int): out_channels of block.
+            num_blocks (int): Number of blocks.
+            stride (int): Stride of the first block.
+            dilation (int): Dilation of the first block.
+            expand_ratio (int): Expand the number of channels of the
+                hidden layer in InvertedResidual by this ratio.
+        """
+        layers = []
+        for i in range(num_blocks):
+            layers.append(
+                InvertedResidual(
+                    self.in_channels,
+                    out_channels,
+                    stride if i == 0 else 1,
+                    expand_ratio=expand_ratio,
+                    dilation=dilation if i == 0 else 1)
+            )
+            self.in_channels = out_channels
+
+        return nn.Sequential(*layers)
+
+    def forward(self, x):
+        x = self.conv1(x)
+
+        outs = []
+        for i, layer_name in enumerate(self.layers):
+            layer = getattr(self, layer_name)
+            x = layer(x)
+            if i in self.out_indices:
+                outs.append(x)
+
+        if len(outs) == 1:
+            return outs[0]
+        else:
+            return tuple(outs)
+
+    def _freeze_stages(self):
+        if self.frozen_stages >= 0:
+            for param in self.conv1.parameters():
+                param.requires_grad = False
+        for i in range(1, self.frozen_stages + 1):
+            layer = getattr(self, f'layer{i}')
+            layer.eval()
+            for param in layer.parameters():
+                param.requires_grad = False

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/mobilenet_v3.py

@@ -0,0 +1,177 @@
+# Modified from mmsegmentation code, referenced from torchvision
+
+import torch
+import torch.nn as nn
+
+from .builder import MODELS
+from .common_models import InvertedResidualV3
+
+
+@MODELS.register()
+class MobileNetV3Encoder(nn.Module):
+    """MobileNetV3 backbone (keep the last 1x1)).
+    This backbone is the improved implementation of `Searching for MobileNetV3
+    <https://ieeexplore.ieee.org/document/9008835>`_.
+    Args:
+        arch (str): Architecture of mobilnetv3, from {'small', 'large'}.
+            Default: 'small'.
+        out_indices (tuple[int]): Output from which layer.
+            Default: (0, 1, 12).
+        frozen_stages (int): Stages to be frozen (all param fixed).
+            Default: -1, which means not freezing any parameters.
+        norm_eval (bool): Whether to set norm layers to eval mode, namely,
+            freeze running stats (mean and var). Note: Effect on Batch Norm
+            and its variants only. Default: False.
+        pretrained (str, optional): model pretrained path. Default: None
+    """
+    # MobileNet V3 for segmentation
+    # Parameters to build each block:
+    #     [kernel size, mid channels, out channels, with_se, act type, stride, dilated]
+    arch_settings = {
+        'small': [[3, 16, 16, True, 'ReLU'],  # block0 layer1 os=4
+                  [3, 72, 24, False, 'ReLU'],  # block1 layer2 os=8
+                  [3, 88, 24, False, 'ReLU'],
+                  [5, 96, 40, True, 'HSwish'],  # block2 layer4 os=16
+                  [5, 240, 40, True, 'HSwish'],
+                  [5, 240, 40, True, 'HSwish'],
+                  [5, 120, 48, True, 'HSwish'],  # block3 layer7 os=16
+                  [5, 144, 48, True, 'HSwish'],
+                  [5, 288, 96, True, 'HSwish'],  # block4 layer9 os=32
+                  [5, 576, 96, True, 'HSwish'],
+                  [5, 576, 96, True, 'HSwish']],
+        'large': [[3, 16, 16, False, 'ReLU'],  # block0 layer1 os=2
+                  [3, 64, 24, False, 'ReLU'],  # block1 layer2 os=4
+                  [3, 72, 24, False, 'ReLU'],
+                  [5, 72, 40, True, 'ReLU'],  # block2 layer4 os=8
+                  [5, 120, 40, True, 'ReLU'],
+                  [5, 120, 40, True, 'ReLU'],
+                  [3, 240, 80, False, 'HSwish'],  # block3 layer7 os=16
+                  [3, 200, 80, False, 'HSwish'],
+                  [3, 184, 80, False, 'HSwish'],
+                  [3, 184, 80, False, 'HSwish'],
+                  [3, 480, 112, True, 'HSwish'],  # block4 layer11 os=16
+                  [3, 672, 112, True, 'HSwish'],
+                  [5, 672, 160, True, 'HSwish'],  # block5 layer13 os=32
+                  [5, 960, 160, True, 'HSwish'],
+                  [5, 960, 160, True, 'HSwish']]
+    }
+
+    def __init__(self, arch='small', out_indices=(12, ), frozen_stages=-1, reduction_factor=1,
+                 norm_eval=False, pretrained=None,
+                 strides=(2, 2, 1, 2, 1, 1, 1, 1, 2, 1, 1),
+                 dilations=(1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2)):
+        super(MobileNetV3Encoder, self).__init__()
+
+        self.pretrained = pretrained
+        assert arch in self.arch_settings
+        assert isinstance(reduction_factor, int) and reduction_factor > 0
+        self.strides = strides
+        self.dilations = dilations
+
+        for index in out_indices:
+            if index not in range(0, len(self.arch_settings[arch]) + 2):
+                raise ValueError(
+                    'the item in out_indices must in '
+                    f'range(0, {len(self.arch_settings[arch]) + 2}). '
+                    f'But received {index}')
+
+        if frozen_stages not in range(-1, len(self.arch_settings[arch]) + 2):
+            raise ValueError('frozen_stages must be in range(-1, '
+                             f'{len(self.arch_settings[arch]) + 2}). '
+                             f'But received {frozen_stages}')
+        self.arch = arch
+        self.out_indices = out_indices
+        self.frozen_stages = frozen_stages
+        self.reduction_factor = reduction_factor
+        self.norm_eval = norm_eval
+        self.layers = self._make_layer()
+
+        if self.pretrained is None:
+            self.weight_initialization()
+        else:
+            self.load_pretrained()
+
+    def _make_layer(self):
+        layers = []
+
+        # build the first layer (layer0)
+        in_channels = 16
+        layer = nn.Sequential(
+            nn.Conv2d(in_channels=3, out_channels=in_channels, kernel_size=3, stride=2, padding=1, bias=False),
+            nn.BatchNorm2d(in_channels),
+            nn.Hardswish()
+        )
+        self.add_module('layer0', layer)
+        layers.append('layer0')
+
+        layer_setting = self.arch_settings[self.arch]
+        for i, params in enumerate(layer_setting):
+            (kernel_size, mid_channels, out_channels, with_se, act) = params
+            stride = self.strides[i]
+            dilation = self.dilations[i]
+
+            if self.arch == 'large' and i >= 12 or self.arch == 'small' and i >= 8:
+                mid_channels = mid_channels // self.reduction_factor
+                out_channels = out_channels // self.reduction_factor
+            layer = InvertedResidualV3(in_channels=in_channels, out_channels=out_channels, mid_channels=mid_channels,
+                                       kernel_size=kernel_size, stride=stride, with_se=with_se, act=act,
+                                       with_expand_conv=(in_channels != mid_channels), dilation=dilation)
+            in_channels = out_channels
+            layer_name = 'layer{}'.format(i + 1)
+            self.add_module(layer_name, layer)
+            layers.append(layer_name)
+
+        # build the last layer
+        # block5 layer12 os=32 for small model
+        # block6 layer16 os=32 for large model
+        out_channels = 576 if self.arch == 'small' else 960
+        layer = nn.Sequential(
+            nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=1, stride=1,
+                      padding=0, bias=False),
+            nn.BatchNorm2d(out_channels),
+            nn.Hardswish()
+        )
+        layer_name = 'layer{}'.format(len(layer_setting) + 1)
+        self.add_module(layer_name, layer)
+        layers.append(layer_name)
+
+        return layers
+
+    def forward(self, x):
+        outs = []
+        for i, layer_name in enumerate(self.layers):
+            layer = getattr(self, layer_name)
+            x = layer(x)
+            if i in self.out_indices:
+                outs.append(x)
+        return outs[-1]
+
+    def _freeze_stages(self):
+        for i in range(self.frozen_stages + 1):
+            layer = getattr(self, f'layer{i}')
+            layer.eval()
+            for param in layer.parameters():
+                param.requires_grad = False
+
+    def load_pretrained(self):
+        state_dict = torch.load(self.pretrained)
+        self_state_dict = self.state_dict()
+        self_keys = list(self_state_dict.keys())
+        for i, (_, v) in enumerate(state_dict.items()):
+            if i > len(self_keys) - 1:
+                break
+            self_state_dict[self_keys[i]] = v
+        self.load_state_dict(self_state_dict)
+
+    def weight_initialization(self):
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                nn.init.kaiming_normal_(m.weight, mode="fan_out")
+                if m.bias is not None:
+                    nn.init.zeros_(m.bias)
+            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):
+                nn.init.ones_(m.weight)
+                nn.init.zeros_(m.bias)
+            elif isinstance(m, nn.Linear):
+                nn.init.normal_(m.weight, 0, 0.01)
+                nn.init.zeros_(m.bias)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/rep_vgg.py

@@ -0,0 +1,340 @@
+# https://github.com/DingXiaoH/RepVGG
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import numpy as np
+
+from .builder import MODELS
+
+
+def conv_bn(in_channels, out_channels, kernel_size, stride, padding, groups=1):
+    result = nn.Sequential()
+    result.add_module('conv', nn.Conv2d(in_channels=in_channels, out_channels=out_channels,
+                                        kernel_size=kernel_size, stride=stride, padding=padding, groups=groups,
+                                        bias=False))
+    result.add_module('bn', nn.BatchNorm2d(num_features=out_channels))
+    return result
+
+
+class SEBlock(nn.Module):
+    # https://openaccess.thecvf.com/content_cvpr_2018/html/Hu_Squeeze-and-Excitation_Networks_CVPR_2018_paper.html
+    def __init__(self, input_channels, internal_neurons):
+        super(SEBlock, self).__init__()
+        self.down = nn.Conv2d(in_channels=input_channels, out_channels=internal_neurons, kernel_size=1, stride=1,
+                              bias=True)
+        self.up = nn.Conv2d(in_channels=internal_neurons, out_channels=input_channels, kernel_size=1, stride=1,
+                            bias=True)
+        self.input_channels = input_channels
+
+    def forward(self, inputs):
+        x = F.avg_pool2d(inputs, kernel_size=inputs.size(3))
+        x = self.down(x)
+        x = F.relu(x)
+        x = self.up(x)
+        x = torch.sigmoid(x)
+        x = x.view(-1, self.input_channels, 1, 1)
+        return inputs * x
+
+
+class RepVGGBlock(nn.Module):
+
+    def __init__(self, in_channels, out_channels, kernel_size,
+                 stride=1, padding=0, dilation=1, groups=1, padding_mode='zeros', deploy=False, use_se=False):
+        super(RepVGGBlock, self).__init__()
+        self.deploy = deploy
+        self.groups = groups
+        self.in_channels = in_channels
+
+        assert kernel_size == 3
+        assert padding == 1
+
+        padding_11 = padding - kernel_size // 2
+
+        self.nonlinearity = nn.ReLU()
+
+        if use_se:
+            self.se = SEBlock(out_channels, internal_neurons=out_channels // 16)
+        else:
+            self.se = nn.Identity()
+
+        if deploy:
+            self.rbr_reparam = nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size,
+                                         stride=stride,
+                                         padding=padding, dilation=dilation, groups=groups, bias=True,
+                                         padding_mode=padding_mode)
+
+        else:
+            self.rbr_identity = nn.BatchNorm2d(
+                num_features=in_channels) if out_channels == in_channels and stride == 1 else None
+            self.rbr_dense = conv_bn(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size,
+                                     stride=stride, padding=padding, groups=groups)
+            self.rbr_1x1 = conv_bn(in_channels=in_channels, out_channels=out_channels, kernel_size=1, stride=stride,
+                                   padding=padding_11, groups=groups)
+            print('RepVGG Block, identity = ', self.rbr_identity)
+
+    def forward(self, inputs):
+        if hasattr(self, 'rbr_reparam'):
+            return self.nonlinearity(self.se(self.rbr_reparam(inputs)))
+
+        if self.rbr_identity is None:
+            id_out = 0
+        else:
+            id_out = self.rbr_identity(inputs)
+
+        return self.nonlinearity(self.se(self.rbr_dense(inputs) + self.rbr_1x1(inputs) + id_out))
+
+    #   Optional. This improves the accuracy and facilitates quantization.
+    #   1.  Cancel the original weight decay on rbr_dense.conv.weight and rbr_1x1.conv.weight.
+    #   2.  Use like this.
+    #       loss = criterion(....)
+    #       for every RepVGGBlock blk:
+    #           loss += weight_decay_coefficient * 0.5 * blk.get_cust_L2()
+    #       optimizer.zero_grad()
+    #       loss.backward()
+    def get_custom_L2(self):
+        K3 = self.rbr_dense.conv.weight
+        K1 = self.rbr_1x1.conv.weight
+        t3 = (self.rbr_dense.bn.weight / ((self.rbr_dense.bn.running_var + self.rbr_dense.bn.eps).sqrt())).reshape(-1,
+                                                                                                                   1, 1,
+                                                                                                                   1).detach()
+        t1 = (self.rbr_1x1.bn.weight / ((self.rbr_1x1.bn.running_var + self.rbr_1x1.bn.eps).sqrt())).reshape(-1, 1, 1,
+                                                                                                             1).detach()
+        # The L2 loss of the "circle" of weights in 3x3 kernel. Use regular L2 on them.
+        l2_loss_circle = (K3 ** 2).sum() - (K3[:, :, 1:2, 1:2] ** 2).sum()
+        # The equivalent resultant central point of 3x3 kernel.
+        eq_kernel = K3[:, :, 1:2, 1:2] * t3 + K1 * t1
+        # Normalize for an L2 coefficient comparable to regular L2.
+        l2_loss_eq_kernel = (eq_kernel ** 2 / (t3 ** 2 + t1 ** 2)).sum()
+        return l2_loss_eq_kernel + l2_loss_circle
+
+    #   This func derives the equivalent kernel and bias in a DIFFERENTIABLE way.
+    #   You can get the equivalent kernel and bias at any time and do whatever you want,
+    #   for example, apply some penalties or constraints during training, just like you do to the other models.
+    #   May be useful for quantization or pruning.
+    def get_equivalent_kernel_bias(self):
+        kernel3x3, bias3x3 = self._fuse_bn_tensor(self.rbr_dense)
+        kernel1x1, bias1x1 = self._fuse_bn_tensor(self.rbr_1x1)
+        kernelid, biasid = self._fuse_bn_tensor(self.rbr_identity)
+        return kernel3x3 + self._pad_1x1_to_3x3_tensor(kernel1x1) + kernelid, bias3x3 + bias1x1 + biasid
+
+    def _pad_1x1_to_3x3_tensor(self, kernel1x1):
+        if kernel1x1 is None:
+            return 0
+        else:
+            return torch.nn.functional.pad(kernel1x1, [1, 1, 1, 1])
+
+    def _fuse_bn_tensor(self, branch):
+        if branch is None:
+            return 0, 0
+        if isinstance(branch, nn.Sequential):
+            kernel = branch.conv.weight
+            running_mean = branch.bn.running_mean
+            running_var = branch.bn.running_var
+            gamma = branch.bn.weight
+            beta = branch.bn.bias
+            eps = branch.bn.eps
+        else:
+            assert isinstance(branch, nn.BatchNorm2d)
+            if not hasattr(self, 'id_tensor'):
+                input_dim = self.in_channels // self.groups
+                kernel_value = np.zeros((self.in_channels, input_dim, 3, 3), dtype=np.float32)
+                for i in range(self.in_channels):
+                    kernel_value[i, i % input_dim, 1, 1] = 1
+                self.id_tensor = torch.from_numpy(kernel_value).to(branch.weight.device)
+            kernel = self.id_tensor
+            running_mean = branch.running_mean
+            running_var = branch.running_var
+            gamma = branch.weight
+            beta = branch.bias
+            eps = branch.eps
+        std = (running_var + eps).sqrt()
+        t = (gamma / std).reshape(-1, 1, 1, 1)
+        return kernel * t, beta - running_mean * gamma / std
+
+    def switch_to_deploy(self):
+        if hasattr(self, 'rbr_reparam'):
+            return
+        kernel, bias = self.get_equivalent_kernel_bias()
+        self.rbr_reparam = nn.Conv2d(in_channels=self.rbr_dense.conv.in_channels,
+                                     out_channels=self.rbr_dense.conv.out_channels,
+                                     kernel_size=self.rbr_dense.conv.kernel_size, stride=self.rbr_dense.conv.stride,
+                                     padding=self.rbr_dense.conv.padding, dilation=self.rbr_dense.conv.dilation,
+                                     groups=self.rbr_dense.conv.groups, bias=True)
+        self.rbr_reparam.weight.data = kernel
+        self.rbr_reparam.bias.data = bias
+        for para in self.parameters():
+            para.detach_()
+        self.__delattr__('rbr_dense')
+        self.__delattr__('rbr_1x1')
+        if hasattr(self, 'rbr_identity'):
+            self.__delattr__('rbr_identity')
+        if hasattr(self, 'id_tensor'):
+            self.__delattr__('id_tensor')
+        self.deploy = True
+
+
+class RepVGG(nn.Module):
+    def __init__(self, num_blocks, num_classes=1000, width_multiplier=None, override_groups_map=None, deploy=False,
+                 use_se=False):
+        super(RepVGG, self).__init__()
+
+        assert len(width_multiplier) == 4
+
+        self.deploy = deploy
+        self.override_groups_map = override_groups_map or dict()
+        self.use_se = use_se
+
+        assert 0 not in self.override_groups_map
+
+        self.in_planes = min(64, int(64 * width_multiplier[0]))
+
+        self.stage0 = RepVGGBlock(in_channels=3, out_channels=self.in_planes, kernel_size=3, stride=2, padding=1,
+                                  deploy=self.deploy, use_se=self.use_se)
+        self.cur_layer_idx = 1
+        self.stage1 = self._make_stage(int(64 * width_multiplier[0]), num_blocks[0], stride=2)
+        self.stage2 = self._make_stage(int(128 * width_multiplier[1]), num_blocks[1], stride=2)
+        self.stage3 = self._make_stage(int(256 * width_multiplier[2]), num_blocks[2], stride=2)
+        self.stage4 = self._make_stage(int(512 * width_multiplier[3]), num_blocks[3], stride=2)
+        self.gap = nn.AdaptiveAvgPool2d(output_size=1)
+        self.linear = nn.Linear(int(512 * width_multiplier[3]), num_classes)
+
+    def _make_stage(self, planes, num_blocks, stride):
+        strides = [stride] + [1] * (num_blocks - 1)
+        blocks = []
+        for stride in strides:
+            cur_groups = self.override_groups_map.get(self.cur_layer_idx, 1)
+            blocks.append(RepVGGBlock(in_channels=self.in_planes, out_channels=planes, kernel_size=3,
+                                      stride=stride, padding=1, groups=cur_groups, deploy=self.deploy,
+                                      use_se=self.use_se))
+            self.in_planes = planes
+            self.cur_layer_idx += 1
+        return nn.Sequential(*blocks)
+
+    def forward(self, x):
+        out = self.stage0(x)
+        out = self.stage1(out)
+        out = self.stage2(out)
+        out = self.stage3(out)
+        out = self.stage4(out)
+        out = self.gap(out)
+        out = out.view(out.size(0), -1)
+        out = self.linear(out)
+        return out
+
+
+optional_groupwise_layers = [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26]
+g2_map = {l: 2 for l in optional_groupwise_layers}
+g4_map = {l: 4 for l in optional_groupwise_layers}
+
+
+def create_RepVGG_A0(deploy=False):
+    return RepVGG(num_blocks=[2, 4, 14, 1], num_classes=1000,
+                  width_multiplier=[0.75, 0.75, 0.75, 2.5], override_groups_map=None, deploy=deploy)
+
+
+def create_RepVGG_A1(deploy=False):
+    return RepVGG(num_blocks=[2, 4, 14, 1], num_classes=1000,
+                  width_multiplier=[1, 1, 1, 2.5], override_groups_map=None, deploy=deploy)
+
+
+def create_RepVGG_A2(deploy=False):
+    return RepVGG(num_blocks=[2, 4, 14, 1], num_classes=1000,
+                  width_multiplier=[1.5, 1.5, 1.5, 2.75], override_groups_map=None, deploy=deploy)
+
+
+def create_RepVGG_B0(deploy=False):
+    return RepVGG(num_blocks=[4, 6, 16, 1], num_classes=1000,
+                  width_multiplier=[1, 1, 1, 2.5], override_groups_map=None, deploy=deploy)
+
+
+def create_RepVGG_B1(deploy=False):
+    return RepVGG(num_blocks=[4, 6, 16, 1], num_classes=1000,
+                  width_multiplier=[2, 2, 2, 4], override_groups_map=None, deploy=deploy)
+
+
+def create_RepVGG_B1g2(deploy=False):
+    return RepVGG(num_blocks=[4, 6, 16, 1], num_classes=1000,
+                  width_multiplier=[2, 2, 2, 4], override_groups_map=g2_map, deploy=deploy)
+
+
+def create_RepVGG_B2(deploy=False):
+    return RepVGG(num_blocks=[4, 6, 16, 1], num_classes=1000,
+                  width_multiplier=[2.5, 2.5, 2.5, 5], override_groups_map=None, deploy=deploy)
+
+
+func_dict = {
+    'RepVGG-A0': create_RepVGG_A0,
+    'RepVGG-A1': create_RepVGG_A1,
+    'RepVGG-A2': create_RepVGG_A2,
+    'RepVGG-B0': create_RepVGG_B0,
+    'RepVGG-B1': create_RepVGG_B1,
+    'RepVGG-B1g2': create_RepVGG_B1g2,
+    'RepVGG-B2': create_RepVGG_B2
+}
+
+pretrained_model_dict = {
+    'RepVGG-A0': 'RepVGG-A0-train.pth',
+    'RepVGG-A1': 'RepVGG-A1-train.pth',
+    'RepVGG-A2': 'RepVGG-A2-train.pth',
+    'RepVGG-B0': 'RepVGG-B0-train.pth',
+    'RepVGG-B1': 'RepVGG-B1-train.pth',
+    'RepVGG-B1g2': 'RepVGG-B1g2-train.pth',
+    'RepVGG-B2': 'RepVGG-B2-train.pth'
+}
+
+
+def get_RepVGG_func_by_name(name):
+    return func_dict[name]
+
+
+@MODELS.register()
+class RepVggEncoder(nn.Module):
+    def __init__(self, backbone_name, pretrained=False, deploy=False):
+        super(RepVggEncoder, self).__init__()
+        repvgg_fn = get_RepVGG_func_by_name(backbone_name)
+        self.encoder = repvgg_fn(deploy)
+        if pretrained:
+            checkpoint = torch.load(pretrained_model_dict[backbone_name])
+            if 'state_dict' in checkpoint:
+                checkpoint = checkpoint['state_dict']
+            ckpt = {k.replace('module.', ''): v for k, v in checkpoint.items()}  # strip the names
+            self.encoder.load_state_dict(ckpt)
+        # TODO: Refactor with feature extractor
+        self.layer0 = self.encoder.stage0
+        self.layer1 = self.encoder.stage1
+        self.layer2 = self.encoder.stage2
+        self.layer3 = self.encoder.stage3
+        self.layer4 = self.encoder.stage4
+
+        #   The last two stages should have stride=1 for semantic segmentation
+        #   Note that the stride of 1x1 should be the same as the 3x3
+        #   Use dilation following the implementation of PSPNet
+        secondlast_channel = 0
+        for n, m in self.layer3.named_modules():
+            if ('rbr_dense' in n or 'rbr_reparam' in n) and isinstance(m, nn.Conv2d):
+                m.dilation, m.padding, m.stride = (2, 2), (2, 2), (1, 1)
+                print('change dilation, padding, stride of ', n)
+                secondlast_channel = m.out_channels
+            elif 'rbr_1x1' in n and isinstance(m, nn.Conv2d):
+                m.stride = (1, 1)
+                print('change stride of ', n)
+        last_channel = 0
+        for n, m in self.layer4.named_modules():
+            if ('rbr_dense' in n or 'rbr_reparam' in n) and isinstance(m, nn.Conv2d):
+                m.dilation, m.padding, m.stride = (4, 4), (4, 4), (1, 1)
+                print('change dilation, padding, stride of ', n)
+                last_channel = m.out_channels
+            elif 'rbr_1x1' in n and isinstance(m, nn.Conv2d):
+                m.stride = (1, 1)
+                print('change stride of ', n)
+        self.fea_dim = last_channel
+
+    def forward(self, x):
+        x0 = self.layer0(x)
+        x1 = self.layer1(x0)
+        x2 = self.layer2(x1)
+        x3 = self.layer3(x2)
+        x4 = self.layer4(x3)
+
+        return x4

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/resnet.py

@@ -0,0 +1,427 @@
+import torch
+import torch.nn as nn
+from torch import Tensor
+from typing import List, Optional
+from .utils import load_state_dict_from_url
+try:
+    from ..common import warnings
+except ImportError:
+    import warnings
+
+
+__all__ = ['ResNet', 'resnet18', 'resnet18_reduced', 'resnet34', 'resnet50', 'resnet101',
+           'resnet152', 'resnext50_32x4d', 'resnext101_32x8d',
+           'wide_resnet50_2', 'wide_resnet101_2']
+
+
+model_urls = {
+    'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth',
+    'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth',
+    'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',
+    'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',
+    'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',
+    'resnext50_32x4d': 'https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth',
+    'resnext101_32x8d': 'https://download.pytorch.org/models/resnext101_32x8d-8ba56ff5.pth',
+    'wide_resnet50_2': 'https://download.pytorch.org/models/wide_resnet50_2-95faca4d.pth',
+    'wide_resnet101_2': 'https://download.pytorch.org/models/wide_resnet101_2-32ee1156.pth',
+}
+
+
+class FrozenBatchNorm2d(torch.nn.Module):
+    """
+    BatchNorm2d where the batch statistics and the affine parameters
+    are fixed
+    """
+
+    def __init__(
+        self,
+        num_features: int,
+        eps: float = 1e-5,
+        n: Optional[int] = None,
+    ):
+        # n=None for backward-compatibility
+        if n is not None:
+            warnings.warn("`n` argument is deprecated and has been renamed `num_features`",
+                          DeprecationWarning)
+            num_features = n
+        super(FrozenBatchNorm2d, self).__init__()
+        self.eps = eps
+        self.register_buffer("weight", torch.ones(num_features))
+        self.register_buffer("bias", torch.zeros(num_features))
+        self.register_buffer("running_mean", torch.zeros(num_features))
+        self.register_buffer("running_var", torch.ones(num_features))
+
+    def _load_from_state_dict(
+        self,
+        state_dict: dict,
+        prefix: str,
+        local_metadata: dict,
+        strict: bool,
+        missing_keys: List[str],
+        unexpected_keys: List[str],
+        error_msgs: List[str],
+    ):
+        num_batches_tracked_key = prefix + 'num_batches_tracked'
+        if num_batches_tracked_key in state_dict:
+            del state_dict[num_batches_tracked_key]
+
+        super(FrozenBatchNorm2d, self)._load_from_state_dict(
+            state_dict, prefix, local_metadata, strict,
+            missing_keys, unexpected_keys, error_msgs)
+
+    def forward(self, x: Tensor) -> Tensor:
+        # move reshapes to the beginning
+        # to make it fuser-friendly
+        w = self.weight.reshape(1, -1, 1, 1)
+        b = self.bias.reshape(1, -1, 1, 1)
+        rv = self.running_var.reshape(1, -1, 1, 1)
+        rm = self.running_mean.reshape(1, -1, 1, 1)
+        scale = w * (rv + self.eps).rsqrt()
+        bias = b - rm * scale
+        return x * scale + bias
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}({self.weight.shape[0]}, eps={self.eps})"
+
+
+def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1):
+    """3x3 convolution with padding"""
+    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
+                     padding=dilation, groups=groups, bias=False, dilation=dilation)
+
+
+def conv1x1(in_planes, out_planes, stride=1):
+    """1x1 convolution"""
+    return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)
+
+
+class BasicBlock(nn.Module):
+    expansion = 1
+
+    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,
+                 base_width=64, dilation=1, norm_layer=None):
+        super(BasicBlock, self).__init__()
+        if norm_layer is None:
+            norm_layer = nn.BatchNorm2d
+        if groups != 1 or base_width != 64:
+            raise ValueError('BasicBlock only supports groups=1 and base_width=64')
+        # if dilation > 1:
+        #     raise NotImplementedError("Dilation > 1 not supported in BasicBlock")
+        # Both self.conv1 and self.downsample layers downsample the input when stride != 1
+        self.conv1 = conv3x3(inplanes, planes, stride, 1, dilation)
+        self.bn1 = norm_layer(planes)
+        self.relu = nn.ReLU(inplace=True)
+        self.conv2 = conv3x3(planes, planes)
+        self.bn2 = norm_layer(planes)
+        self.downsample = downsample
+        self.stride = stride
+
+    def forward(self, x):
+        identity = x
+
+        out = self.conv1(x)
+        out = self.bn1(out)
+        out = self.relu(out)
+
+        out = self.conv2(out)
+        out = self.bn2(out)
+
+        if self.downsample is not None:
+            identity = self.downsample(x)
+
+        out += identity
+        out = self.relu(out)
+
+        return out
+
+
+class Bottleneck(nn.Module):
+    # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2)
+    # while original implementation places the stride at the first 1x1 convolution(self.conv1)
+    # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385.
+    # This variant is also known as ResNet V1.5 and improves accuracy according to
+    # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch.
+
+    expansion = 4
+
+    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,
+                 base_width=64, dilation=1, norm_layer=None):
+        super(Bottleneck, self).__init__()
+        if norm_layer is None:
+            norm_layer = nn.BatchNorm2d
+        width = int(planes * (base_width / 64.)) * groups
+        # Both self.conv2 and self.downsample layers downsample the input when stride != 1
+        self.conv1 = conv1x1(inplanes, width)
+        self.bn1 = norm_layer(width)
+        self.conv2 = conv3x3(width, width, stride, groups, dilation)
+        self.bn2 = norm_layer(width)
+        self.conv3 = conv1x1(width, planes * self.expansion)
+        self.bn3 = norm_layer(planes * self.expansion)
+        self.relu = nn.ReLU(inplace=True)
+        self.downsample = downsample
+        self.stride = stride
+
+    def forward(self, x):
+        identity = x
+
+        out = self.conv1(x)
+        out = self.bn1(out)
+        out = self.relu(out)
+
+        out = self.conv2(out)
+        out = self.bn2(out)
+        out = self.relu(out)
+
+        out = self.conv3(out)
+        out = self.bn3(out)
+
+        if self.downsample is not None:
+            identity = self.downsample(x)
+
+        out += identity
+        out = self.relu(out)
+
+        return out
+
+
+class ResNet(nn.Module):
+    def __init__(self, block, layers, num_classes=1000, zero_init_residual=False,
+                 groups=1, width_per_group=64, replace_stride_with_dilation=None,
+                 norm_layer=None, channels=None):
+        super(ResNet, self).__init__()
+        if channels is None:  # To keep BC
+            channels = [64, 128, 256, 512]
+        if norm_layer is None:
+            norm_layer = nn.BatchNorm2d
+        self._norm_layer = norm_layer
+
+        self.inplanes = channels[0]
+        self.dilation = 1
+        if replace_stride_with_dilation is None:
+            # each element in the tuple indicates if we should replace
+            # the 2x2 stride with a dilated convolution instead
+            replace_stride_with_dilation = [False, False, False]
+        if len(replace_stride_with_dilation) != 3:
+            raise ValueError("replace_stride_with_dilation should be None "
+                             "or a 3-element tuple, got {}".format(replace_stride_with_dilation))
+        self.groups = groups
+        self.base_width = width_per_group
+        self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3,
+                               bias=False)
+        self.bn1 = norm_layer(self.inplanes)
+        self.relu = nn.ReLU(inplace=True)
+        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
+        self.layer1 = self._make_layer(block, channels[0], layers[0])
+        self.layer2 = self._make_layer(block, channels[1], layers[1], stride=2,
+                                       dilate=replace_stride_with_dilation[0])
+        self.layer3 = self._make_layer(block, channels[2], layers[2], stride=2,
+                                       dilate=replace_stride_with_dilation[1])
+        self.layer4 = self._make_layer(block, channels[3], layers[3], stride=2,
+                                       dilate=replace_stride_with_dilation[2])
+        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
+        self.fc = nn.Linear(channels[3] * block.expansion, num_classes)
+
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
+            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):
+                nn.init.constant_(m.weight, 1)
+                nn.init.constant_(m.bias, 0)
+
+        # Zero-initialize the last BN in each residual branch,
+        # so that the residual branch starts with zeros, and each residual block behaves like an identity.
+        # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677
+        if zero_init_residual:
+            for m in self.modules():
+                if isinstance(m, Bottleneck):
+                    nn.init.constant_(m.bn3.weight, 0)
+                elif isinstance(m, BasicBlock):
+                    nn.init.constant_(m.bn2.weight, 0)
+
+    def _make_layer(self, block, planes, blocks, stride=1, dilate=False):
+        norm_layer = self._norm_layer
+        downsample = None
+        previous_dilation = self.dilation
+        if dilate:
+            self.dilation *= stride
+            stride = 1
+        if stride != 1 or self.inplanes != planes * block.expansion:
+            downsample = nn.Sequential(
+                conv1x1(self.inplanes, planes * block.expansion, stride),
+                norm_layer(planes * block.expansion),
+            )
+
+        layers = []
+        layers.append(block(self.inplanes, planes, stride, downsample, self.groups,
+                            self.base_width, previous_dilation, norm_layer))
+        self.inplanes = planes * block.expansion
+        for _ in range(1, blocks):
+            layers.append(block(self.inplanes, planes, groups=self.groups,
+                                base_width=self.base_width, dilation=self.dilation,
+                                norm_layer=norm_layer))
+
+        return nn.Sequential(*layers)
+
+    def forward(self, x):
+        x = self.conv1(x)
+        x = self.bn1(x)
+        x = self.relu(x)
+        x = self.maxpool(x)
+
+        x = self.layer1(x)
+        x = self.layer2(x)
+        x = self.layer3(x)
+        x = self.layer4(x)
+
+        x = self.avgpool(x)
+        x = torch.flatten(x, 1)
+        x = self.fc(x)
+
+        return x
+
+
+def _resnet(arch, block, layers, pretrained, progress, **kwargs):
+    model = ResNet(block, layers, **kwargs)
+    if pretrained:
+        # TODO: load for reduced ResNets
+        state_dict = load_state_dict_from_url(model_urls[arch],
+                                              progress=progress)
+        model.load_state_dict(state_dict)
+        print('Loaded torchvision ImageNet pre-trained weights V1.')
+    return model
+
+
+def resnet18(pretrained=False, progress=True, **kwargs):
+    r"""ResNet-18 model from
+    `"Deep Residual Learning for Image Recognition" <https://arxiv.org/pdf/1512.03385.pdf>`_
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+    """
+    return _resnet('resnet18', BasicBlock, [2, 2, 2, 2], pretrained, progress,
+                   **kwargs)
+
+
+def resnet18_reduced(pretrained=False, progress=True, expansion=1, **kwargs):
+    r"""Reduced ResNet18 (LSTR modification, a 4x expansion yields some kind of ResNet-17)
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+        expansion (int): Expansion rate for reduced ResNet18
+    """
+    channels = [channel * expansion for channel in [16, 32, 64, 128]]
+    return _resnet('resnet18', BasicBlock, [1, 2, 2, 2], pretrained, progress, channels=channels, **kwargs)
+
+
+def resnet34(pretrained=False, progress=True, **kwargs):
+    r"""ResNet-34 model from
+    `"Deep Residual Learning for Image Recognition" <https://arxiv.org/pdf/1512.03385.pdf>`_
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+    """
+    return _resnet('resnet34', BasicBlock, [3, 4, 6, 3], pretrained, progress,
+                   **kwargs)
+
+
+def resnet50(pretrained=False, progress=True, **kwargs):
+    r"""ResNet-50 model from
+    `"Deep Residual Learning for Image Recognition" <https://arxiv.org/pdf/1512.03385.pdf>`_
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+    """
+    return _resnet('resnet50', Bottleneck, [3, 4, 6, 3], pretrained, progress,
+                   **kwargs)
+
+
+def resnet101(pretrained=False, progress=True, **kwargs):
+    r"""ResNet-101 model from
+    `"Deep Residual Learning for Image Recognition" <https://arxiv.org/pdf/1512.03385.pdf>`_
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+    """
+    return _resnet('resnet101', Bottleneck, [3, 4, 23, 3], pretrained, progress,
+                   **kwargs)
+
+
+def resnet152(pretrained=False, progress=True, **kwargs):
+    r"""ResNet-152 model from
+    `"Deep Residual Learning for Image Recognition" <https://arxiv.org/pdf/1512.03385.pdf>`_
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+    """
+    return _resnet('resnet152', Bottleneck, [3, 8, 36, 3], pretrained, progress,
+                   **kwargs)
+
+
+def resnext50_32x4d(pretrained=False, progress=True, **kwargs):
+    r"""ResNeXt-50 32x4d model from
+    `"Aggregated Residual Transformation for Deep Neural Networks" <https://arxiv.org/pdf/1611.05431.pdf>`_
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+    """
+    kwargs['groups'] = 32
+    kwargs['width_per_group'] = 4
+    return _resnet('resnext50_32x4d', Bottleneck, [3, 4, 6, 3],
+                   pretrained, progress, **kwargs)
+
+
+def resnext101_32x8d(pretrained=False, progress=True, **kwargs):
+    r"""ResNeXt-101 32x8d model from
+    `"Aggregated Residual Transformation for Deep Neural Networks" <https://arxiv.org/pdf/1611.05431.pdf>`_
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+    """
+    kwargs['groups'] = 32
+    kwargs['width_per_group'] = 8
+    return _resnet('resnext101_32x8d', Bottleneck, [3, 4, 23, 3],
+                   pretrained, progress, **kwargs)
+
+
+def wide_resnet50_2(pretrained=False, progress=True, **kwargs):
+    r"""Wide ResNet-50-2 model from
+    `"Wide Residual Networks" <https://arxiv.org/pdf/1605.07146.pdf>`_
+
+    The model is the same as ResNet except for the bottleneck number of channels
+    which is twice larger in every block. The number of channels in outer 1x1
+    convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048
+    channels, and in Wide ResNet-50-2 has 2048-1024-2048.
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+    """
+    kwargs['width_per_group'] = 64 * 2
+    return _resnet('wide_resnet50_2', Bottleneck, [3, 4, 6, 3],
+                   pretrained, progress, **kwargs)
+
+
+def wide_resnet101_2(pretrained=False, progress=True, **kwargs):
+    r"""Wide ResNet-101-2 model from
+    `"Wide Residual Networks" <https://arxiv.org/pdf/1605.07146.pdf>`_
+
+    The model is the same as ResNet except for the bottleneck number of channels
+    which is twice larger in every block. The number of channels in outer 1x1
+    convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048
+    channels, and in Wide ResNet-50-2 has 2048-1024-2048.
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+    """
+    kwargs['width_per_group'] = 64 * 2
+    return _resnet('wide_resnet101_2', Bottleneck, [3, 4, 23, 3],
+                   pretrained, progress, **kwargs)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/segmentation/__init__.py

@@ -0,0 +1,7 @@
+from .segmentation import *
+from .fcn import *
+from .deeplab import *
+from .deeplab_vgg import DeepLabV1, DeepLabV1Lane, SegRepVGG
+from .erfnet import ERFNet
+from .enet import ENet
+from .simple_seg_head import SimpleSegHead

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/segmentation/_utils.py

@@ -0,0 +1,70 @@
+import torch.nn as nn
+from torch import load
+from collections import OrderedDict
+
+
+class _EncoderDecoderModel(nn.Module):
+    def __init__(self):
+        super().__init__()
+
+    def _load_encoder(self, pretrained_weights):
+        if pretrained_weights is not None:  # Load pre-trained weights
+            try:
+                saved_weights = load(pretrained_weights)['model']
+            except FileNotFoundError:
+                raise FileNotFoundError('pretrained_weights is not there! '
+                                        'Please set pretrained_weights=None if you are only testing.')
+            original_weights = self.state_dict()
+            for key in saved_weights.keys():
+                if key in original_weights.keys():
+                    original_weights[key] = saved_weights[key]
+            self.load_state_dict(original_weights)
+        else:
+            print('No pre-training.')
+
+    def forward(self, x):
+        pass
+
+
+class _SimpleSegmentationModel(nn.Module):
+    def __init__(self, backbone, classifier, aux_classifier=None, recon_head=None,
+                 lane_classifier=None, channel_reducer=None, scnn_layer=None):
+        super(_SimpleSegmentationModel, self).__init__()
+        self.backbone = backbone
+        self.classifier = classifier
+        self.aux_classifier = aux_classifier  # Name aux is already take for COCO pre-trained models
+        self.lane_classifier = lane_classifier
+        self.channel_reducer = channel_reducer  # Reduce ResNet feature channels to 128 as did in RESA
+        self.scnn_layer = scnn_layer
+        self.recon_head = recon_head
+
+    def forward(self, x):
+        # input_shape = x.shape[-2:]
+        # contract: features is a dict of tensors
+        features = self.backbone(x)
+        result = OrderedDict()
+        x = features['out']
+
+        # For lane detection
+        if self.channel_reducer is not None:
+            x = self.channel_reducer(x)
+        if self.scnn_layer is not None:
+            x = self.scnn_layer(x)
+
+        # Semantic segmentation
+        x = self.classifier(x)
+        # x = F.interpolate(x, size=(513, 513), mode='bilinear', align_corners=True)
+        result['out'] = x
+
+        # For lane detection
+        if self.lane_classifier is not None:
+            result['lane'] = self.lane_classifier(x.softmax(dim=1))
+
+        # For COCO pre-trained models
+        if self.aux_classifier is not None:
+            x = features['aux']
+            x = self.aux_classifier(x)
+            # x = F.interpolate(x, size=input_shape, mode='bilinear', align_corners=True)
+            result['aux'] = x
+
+        return result

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/segmentation/deeplab.py

@@ -0,0 +1,120 @@
+# Note that this file only implement DeepLabs with ResNet backbone,
+# For the original DeepLab-LargeFOV with VGG backbone, refer to deeplab_vgg.py
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+from ..builder import MODELS
+
+
+@MODELS.register()
+class DeepLabV3Head(nn.Sequential):
+    def __init__(self, in_channels, num_classes):
+        super(DeepLabV3Head, self).__init__(
+            ASPP(in_channels, [12, 24, 36]),
+            nn.Conv2d(256, 256, 3, padding=1, bias=False),
+            nn.BatchNorm2d(256),
+            nn.ReLU(),
+            nn.Conv2d(256, num_classes, 1)
+        )
+
+
+# For better format consistency
+@MODELS.register()
+class DeepLabV2Head(nn.Sequential):
+    def __init__(self, in_channels, num_classes):
+        super(DeepLabV2Head, self).__init__(
+            ASPP_V2(in_channels, num_classes, [6, 12, 18, 24])
+        )
+
+
+# For better format consistency
+# Not the official VGG backbone version
+@MODELS.register()
+class DeepLabV1Head(nn.Sequential):
+    def __init__(self, in_channels, num_classes, dilation=12):
+        super(DeepLabV1Head, self).__init__(
+            LargeFOV(in_channels, num_classes, dilation)
+        )
+
+
+class ASPPConv(nn.Sequential):
+    def __init__(self, in_channels, out_channels, dilation):
+        modules = [
+            nn.Conv2d(in_channels, out_channels, 3, padding=dilation, dilation=dilation, bias=False),
+            nn.BatchNorm2d(out_channels),
+            nn.ReLU()
+        ]
+        super(ASPPConv, self).__init__(*modules)
+
+
+class ASPPPooling(nn.Sequential):
+    def __init__(self, in_channels, out_channels):
+        super(ASPPPooling, self).__init__(
+            nn.AdaptiveAvgPool2d(1),
+            nn.Conv2d(in_channels, out_channels, 1, bias=False),
+            nn.BatchNorm2d(out_channels),
+            nn.ReLU())
+
+    def forward(self, x):
+        size = x.shape[-2:]
+        x = super(ASPPPooling, self).forward(x)
+        return F.interpolate(x, size=size, mode='bilinear', align_corners=True)
+
+
+class ASPP(nn.Module):
+    def __init__(self, in_channels, atrous_rates):
+        super(ASPP, self).__init__()
+        out_channels = 256
+        modules = []
+        modules.append(nn.Sequential(
+            nn.Conv2d(in_channels, out_channels, 1, bias=False),
+            nn.BatchNorm2d(out_channels),
+            nn.ReLU()))
+
+        rate1, rate2, rate3 = tuple(atrous_rates)
+        modules.append(ASPPConv(in_channels, out_channels, rate1))
+        modules.append(ASPPConv(in_channels, out_channels, rate2))
+        modules.append(ASPPConv(in_channels, out_channels, rate3))
+        modules.append(ASPPPooling(in_channels, out_channels))
+
+        self.convs = nn.ModuleList(modules)
+
+        self.project = nn.Sequential(
+            nn.Conv2d(5 * out_channels, out_channels, 1, bias=False),
+            nn.BatchNorm2d(out_channels),
+            nn.ReLU(),
+            nn.Dropout(0.5))
+
+    def forward(self, x):
+        res = []
+        for conv in self.convs:
+            res.append(conv(x))
+        res = torch.cat(res, dim=1)
+        return self.project(res)
+
+
+# Copied and modified from hfslyc/AdvSemiSeg/model
+class ASPP_V2(nn.Module):
+    def __init__(self, in_channels, num_classes, atrous_rates):
+        super(ASPP_V2, self).__init__()
+        self.convs = nn.ModuleList()
+        for rates in atrous_rates:
+            self.convs.append(
+                nn.Conv2d(in_channels, num_classes, kernel_size=3, stride=1, padding=rates, dilation=rates, bias=True))
+
+    def forward(self, x):
+        res = self.convs[0](x)
+        for i in range(len(self.convs) - 1):
+            res += self.convs[i + 1](x)
+            return res
+
+
+class LargeFOV(nn.Module):
+    def __init__(self, in_channels, num_classes, dilation=12):
+        super(LargeFOV, self).__init__()
+        self.conv = nn.Conv2d(in_channels, num_classes, kernel_size=3, stride=1,
+                              padding=dilation, dilation=dilation, bias=True)
+
+    def forward(self, x):
+        return self.conv(x)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/segmentation/deeplab_vgg.py

@@ -0,0 +1,103 @@
+# Referenced from jcdubron/scnn_pytorch
+# DeepLabV1 and variants
+import torch.nn as nn
+from collections import OrderedDict
+
+from ..builder import MODELS
+
+
+@MODELS.register()
+class DeepLabV1(nn.Module):
+    # Original DeepLabV1 with VGG-16 for lane detection
+    def __init__(self, backbone_cfg,
+                 num_classes,
+                 spatial_conv_cfg=None,
+                 lane_classifier_cfg=None,
+                 dropout_1=0.1):
+        super(DeepLabV1, self).__init__()
+
+        self.encoder = MODELS.from_dict(backbone_cfg)
+        self.fc67 = nn.Sequential(
+            nn.Conv2d(512, 1024, 3, padding=4, dilation=4, bias=False),
+            nn.BatchNorm2d(1024),
+            nn.ReLU(),
+            nn.Conv2d(1024, 128, 1, bias=False),
+            nn.BatchNorm2d(128),
+            nn.ReLU()
+        )
+        self.scnn = MODELS.from_dict(spatial_conv_cfg)
+        self.fc8 = nn.Sequential(
+            nn.Dropout2d(dropout_1),
+            nn.Conv2d(128, num_classes, 1)
+        )
+        self.softmax = nn.Softmax(dim=1)
+        self.lane_classifier = MODELS.from_dict(lane_classifier_cfg)
+
+    def forward(self, x):
+        out = OrderedDict()
+
+        output = self.encoder(x)
+        output = self.fc67(output)
+
+        if self.scnn is not None:
+            output = self.scnn(output)
+
+        output = self.fc8(output)
+        out['out'] = output
+        if self.lane_classifier is not None:
+            output = self.softmax(output)
+            out['lane'] = self.lane_classifier(output)
+
+        return out
+
+
+@MODELS.register()
+class DeepLabV1Lane(nn.Module):
+    # General lane baseline
+    def __init__(self,
+                 backbone_cfg=None,
+                 spatial_conv_cfg=None,
+                 lane_classifier_cfg=None,
+                 reducer_cfg=None,
+                 classifier_cfg=None,
+                 uper_cfg=None):
+        super().__init__()
+        self.encoder = MODELS.from_dict(backbone_cfg)
+        self.reducer = MODELS.from_dict(reducer_cfg)
+        self.scnn = MODELS.from_dict(spatial_conv_cfg)
+        self.classifier = MODELS.from_dict(classifier_cfg)
+        self.softmax = nn.Softmax(dim=1)
+        self.lane_classifier = MODELS.from_dict(lane_classifier_cfg)
+        self.uper_decoder = MODELS.from_dict(uper_cfg)
+
+    def forward(self, input):
+        out = OrderedDict()
+        output = self.encoder(input)
+        if self.uper_decoder is not None:
+            output = self.uper_decoder(output)
+        if self.reducer is not None:
+            output = self.reducer(output)
+        if self.scnn is not None:
+            output = self.scnn(output)
+        output = self.classifier(output)
+        out['out'] = output
+        if self.lane_classifier is not None:
+            output = self.softmax(output)
+            out['lane'] = self.lane_classifier(output)
+        return out
+
+
+@MODELS.register()
+class SegRepVGG(DeepLabV1Lane):
+    def eval(self, profiling=False):
+        """A secure copy of eval()
+        """
+
+        if profiling:
+            for module in self.encoder.modules():
+                if hasattr(module, 'switch_to_deploy'):
+                    module.switch_to_deploy()
+            print('Deploy!')
+            return self.train(False)
+        else:
+            return self.train(False)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/segmentation/enet.py

@@ -0,0 +1,700 @@
+# copy from https://github.com/davidtvs/PyTorch-ENet/blob/master/models/enet.py
+from collections import OrderedDict
+import torch.nn as nn
+import torch
+from torch.nn import functional as F
+from torch.nn.parameter import Parameter
+
+from ._utils import _EncoderDecoderModel
+from ..builder import MODELS
+
+
+class InitialBlock(nn.Module):
+    """The initial block is composed of two branches:
+    1. a main branch which performs a regular convolution with stride 2;
+    2. an extension branch which performs max-pooling.
+    Doing both operations in parallel and concatenating their results
+    allows for efficient downsampling and expansion. The main branch
+    outputs 13 feature maps while the extension branch outputs 3, for a
+    total of 16 feature maps after concatenation.
+    Keyword arguments:
+    - in_channels (int): the number of input channels.
+    - out_channels (int): the number output channels.
+    - kernel_size (int, optional): the kernel size of the filters used in
+    the convolution layer. Default: 3.
+    - padding (int, optional): zero-padding added to both sides of the
+    input. Default: 0.
+    - bias (bool, optional): Adds a learnable bias to the output if
+    ``True``. Default: False.
+    - relu (bool, optional): When ``True`` ReLU is used as the activation
+    function; otherwise, PReLU is used. Default: True.
+    """
+
+    def __init__(self, in_channels, out_channels, bias=False, relu=True):
+        super().__init__()
+
+        if relu:
+            activation = nn.ReLU
+        else:
+            activation = nn.PReLU
+
+        # Main branch - As stated above the number of output channels for this
+        # branch is the total minus 3, since the remaining channels come from
+        # the extension branch
+        self.main_branch = nn.Conv2d(in_channels, out_channels - 3, kernel_size=3, stride=2, padding=1, bias=bias)
+
+        # Extension branch
+        self.ext_branch = nn.MaxPool2d(3, stride=2, padding=1)
+
+        # Initialize batch normalization to be used after concatenation
+        self.batch_norm = nn.BatchNorm2d(out_channels)
+
+        # PReLU layer to apply after concatenating the branches
+        self.out_activation = activation()
+
+    def forward(self, x):
+        main = self.main_branch(x)
+        ext = self.ext_branch(x)
+
+        # Concatenate branches
+        out = torch.cat((main, ext), 1)
+
+        # Apply batch normalization
+        out = self.batch_norm(out)
+
+        return self.out_activation(out)
+
+
+class RegularBottleneck(nn.Module):
+    """Regular bottlenecks are the main building block of ENet.
+    Main branch:
+    1. Shortcut connection.
+    Extension branch:
+    1. 1x1 convolution which decreases the number of channels by
+    ``internal_ratio``, also called a projection;
+    2. regular, dilated or asymmetric convolution;
+    3. 1x1 convolution which increases the number of channels back to
+    ``channels``, also called an expansion;
+    4. dropout as a regularizer.
+    Keyword arguments:
+    - channels (int): the number of input and output channels.
+    - internal_ratio (int, optional): a scale factor applied to
+    ``channels`` used to compute the number of
+    channels after the projection. eg. given ``channels`` equal to 128 and
+    internal_ratio equal to 2 the number of channels after the projection
+    is 64. Default: 4.
+    - kernel_size (int, optional): the kernel size of the filters used in
+    the convolution layer described above in item 2 of the extension
+    branch. Default: 3.
+    - padding (int, optional): zero-padding added to both sides of the
+    input. Default: 0.
+    - dilation (int, optional): spacing between kernel elements for the
+    convolution described in item 2 of the extension branch. Default: 1.
+    asymmetric (bool, optional): flags if the convolution described in
+    item 2 of the extension branch is asymmetric or not. Default: False.
+    - dropout_prob (float, optional): probability of an element to be
+    zeroed. Default: 0 (no dropout).
+    - bias (bool, optional): Adds a learnable bias to the output if
+    ``True``. Default: False.
+    - relu (bool, optional): When ``True`` ReLU is used as the activation
+    function; otherwise, PReLU is used. Default: True.
+    """
+
+    def __init__(self, channels, internal_ratio=4, kernel_size=3, padding=0, dilation=1, asymmetric=False,
+                 dropout_prob=0.0, is_dropout=True, bias=False, relu=True):
+        super().__init__()
+
+        # Check in the internal_scale parameter is within the expected range
+        # [1, channels]
+        if internal_ratio <= 1 or internal_ratio > channels:
+            raise RuntimeError("Value out of range. Expected value in the "
+                               "interval [1, {0}], got internal_scale={1}."
+                               .format(channels, internal_ratio))
+
+        internal_channels = channels // internal_ratio
+        self.is_dropout = is_dropout
+        if relu:
+            activation = nn.ReLU
+        else:
+            activation = nn.PReLU
+
+        # Main branch - shortcut connection
+
+        # Extension branch - 1x1 convolution, followed by a regular, dilated or
+        # asymmetric convolution, followed by another 1x1 convolution, and,
+        # finally, a regularizer (spatial dropout). Number of channels is constant.
+
+        # 1x1 projection convolution
+        self.ext_conv1 = nn.Sequential(
+            nn.Conv2d(channels, internal_channels, kernel_size=1, stride=1, bias=bias),
+            nn.BatchNorm2d(internal_channels),
+            activation()
+        )
+
+        # If the convolution is asymmetric we split the main convolution in
+        # two. Eg. for a 5x5 asymmetric convolution we have two convolution:
+        # the first is 5x1 and the second is 1x5.
+        if asymmetric:
+            self.ext_conv2 = nn.Sequential(
+                nn.Conv2d(internal_channels, internal_channels, kernel_size=(kernel_size, 1), stride=1,
+                          padding=(padding, 0), dilation=dilation, bias=bias),
+                nn.BatchNorm2d(internal_channels),
+                activation(),
+                nn.Conv2d(internal_channels, internal_channels, kernel_size=(1, kernel_size), stride=1,
+                          padding=(0, padding), dilation=dilation, bias=bias),
+                nn.BatchNorm2d(internal_channels),
+                activation())
+        else:
+            self.ext_conv2 = nn.Sequential(
+                nn.Conv2d(internal_channels, internal_channels, kernel_size=kernel_size, stride=1, padding=padding,
+                          dilation=dilation, bias=bias),
+                nn.BatchNorm2d(internal_channels),
+                activation())
+
+        # 1x1 expansion convolution
+        self.ext_conv3 = nn.Sequential(
+            nn.Conv2d(internal_channels, channels, kernel_size=1, stride=1, bias=bias),
+            nn.BatchNorm2d(channels),
+            # activation()
+        )
+
+        self.ext_regul = nn.Dropout2d(p=dropout_prob)
+
+        # PReLU layer to apply after adding the branches
+        self.out_activation = activation()
+
+    def forward(self, x):
+        # Main branch shortcut
+        main = x
+
+        # Extension branch
+        ext = self.ext_conv1(x)
+        ext = self.ext_conv2(ext)
+        ext = self.ext_conv3(ext)
+        if self.is_dropout:
+            ext = self.ext_regul(ext)
+
+        # Add main and extension branches
+        out = main + ext
+
+        return self.out_activation(out)
+
+
+class DownsamplingBottleneck(nn.Module):
+    """Downsampling bottlenecks further downsample the feature map size.
+    Main branch:
+    1. max pooling with stride 2; indices are saved to be used for
+    unpooling later.
+    Extension branch:
+    1. 2x2 convolution with stride 2 that decreases the number of channels
+    by ``internal_ratio``, also called a projection;
+    2. regular convolution (by default, 3x3);
+    3. 1x1 convolution which increases the number of channels to
+    ``out_channels``, also called an expansion;
+    4. dropout as a regularizer.
+    Keyword arguments:
+    - in_channels (int): the number of input channels.
+    - out_channels (int): the number of output channels.
+    - internal_ratio (int, optional): a scale factor applied to ``channels``
+    used to compute the number of channels after the projection. eg. given
+    ``channels`` equal to 128 and internal_ratio equal to 2 the number of
+    channels after the projection is 64. Default: 4.
+    - return_indices (bool, optional):  if ``True``, will return the max
+    indices along with the outputs. Useful when unpooling later.
+    - dropout_prob (float, optional): probability of an element to be
+    zeroed. Default: 0 (no dropout).
+    - bias (bool, optional): Adds a learnable bias to the output if
+    ``True``. Default: False.
+    - relu (bool, optional): When ``True`` ReLU is used as the activation
+    function; otherwise, PReLU is used. Default: True.
+    """
+
+    def __init__(self, in_channels, out_channels, internal_ratio=4, return_indices=False, dropout_prob=0.0,
+                 is_dropout=True, bias=False,
+                 relu=True):
+        super().__init__()
+
+        # Store parameters that are needed later
+        self.return_indices = return_indices
+        self.is_dropout = is_dropout
+        # Check in the internal_scale parameter is within the expected range
+        # [1, channels]
+        if internal_ratio <= 1 or internal_ratio > in_channels:
+            raise RuntimeError("Value out of range. Expected value in the "
+                               "interval [1, {0}], got internal_scale={1}. "
+                               .format(in_channels, internal_ratio))
+
+        internal_channels = in_channels // internal_ratio
+
+        if relu:
+            activation = nn.ReLU
+        else:
+            activation = nn.PReLU
+
+        # Main branch - max pooling followed by feature map (channels) padding
+        self.main_max1 = nn.MaxPool2d(2, stride=2, return_indices=return_indices)
+
+        # Extension branch - 2x2 convolution, followed by a regular, dilated or
+        # asymmetric convolution, followed by another 1x1 convolution. Number
+        # of channels is doubled.
+
+        # 2x2 projection convolution with stride 2
+        self.ext_conv1 = nn.Sequential(
+            nn.Conv2d(in_channels, internal_channels, kernel_size=2, stride=2, bias=bias),
+            nn.BatchNorm2d(internal_channels),
+            activation())
+
+        # Convolution
+        self.ext_conv2 = nn.Sequential(
+            nn.Conv2d(internal_channels, internal_channels, kernel_size=3, stride=1, padding=1, bias=bias),
+            nn.BatchNorm2d(internal_channels),
+            activation())
+
+        # 1x1 expansion convolution
+        self.ext_conv3 = nn.Sequential(
+            nn.Conv2d(internal_channels, out_channels, kernel_size=1, stride=1, bias=bias),
+            nn.BatchNorm2d(out_channels),
+            # activation()
+        )
+
+        self.ext_regul = nn.Dropout2d(p=dropout_prob)
+
+        # PReLU layer to apply after concatenating the branches
+        self.out_activation = activation()
+
+    def forward(self, x):
+        # Main branch shortcut
+        if self.return_indices:
+            main, max_indices = self.main_max1(x)
+        else:
+            main = self.main_max1(x)
+
+        # Extension branch
+        ext = self.ext_conv1(x)
+        ext = self.ext_conv2(ext)
+        ext = self.ext_conv3(ext)
+        if self.is_dropout:
+            ext = self.ext_regul(ext)
+
+        # Main branch channel padding
+        n, ch_ext, h, w = ext.size()
+        ch_main = main.size()[1]
+        padding = torch.zeros(n, ch_ext - ch_main, h, w)
+
+        # Before concatenating, check if main is on the CPU or GPU and
+        # convert padding accordingly
+        if main.is_cuda:
+            padding = padding.cuda()
+
+        # Concatenate
+        main = torch.cat((main, padding), 1)
+
+        # Add main and extension branches
+        out = main + ext
+
+        return self.out_activation(out), max_indices
+
+
+class UpsamplingBottleneck(nn.Module):
+    """The upsampling bottlenecks upsample the feature map resolution using max
+    pooling indices stored from the corresponding downsampling bottleneck.
+    Main branch:
+    1. 1x1 convolution with stride 1 that decreases the number of channels by
+    ``internal_ratio``, also called a projection;
+    2. max unpool layer using the max pool indices from the corresponding
+    downsampling max pool layer.
+    Extension branch:
+    1. 1x1 convolution with stride 1 that decreases the number of channels by
+    ``internal_ratio``, also called a projection;
+    2. transposed convolution (by default, 3x3);
+    3. 1x1 convolution which increases the number of channels to
+    ``out_channels``, also called an expansion;
+    4. dropout as a regularizer.
+    Keyword arguments:
+    - in_channels (int): the number of input channels.
+    - out_channels (int): the number of output channels.
+    - internal_ratio (int, optional): a scale factor applied to ``in_channels``
+     used to compute the number of channels after the projection. eg. given
+     ``in_channels`` equal to 128 and ``internal_ratio`` equal to 2 the number
+     of channels after the projection is 64. Default: 4.
+    - dropout_prob (float, optional): probability of an element to be zeroed.
+    Default: 0 (no dropout).
+    - bias (bool, optional): Adds a learnable bias to the output if ``True``.
+    Default: False.
+    - relu (bool, optional): When ``True`` ReLU is used as the activation
+    function; otherwise, PReLU is used. Default: True.
+    """
+
+    def __init__(self, in_channels, out_channels, internal_ratio=4, dropout_prob=0.0, is_dropout=True, bias=False,
+                 relu=True):
+        super().__init__()
+
+        # Check in the internal_scale parameter is within the expected range
+        # [1, channels]
+        if internal_ratio <= 1 or internal_ratio > in_channels:
+            raise RuntimeError("Value out of range. Expected value in the "
+                               "interval [1, {0}], got internal_scale={1}. "
+                               .format(in_channels, internal_ratio))
+
+        internal_channels = in_channels // internal_ratio
+        self.is_dropout = is_dropout
+        if relu:
+            activation = nn.ReLU
+        else:
+            activation = nn.PReLU
+
+        # Main branch - max pooling followed by feature map (channels) padding
+        self.main_conv1 = nn.Sequential(
+            nn.Conv2d(in_channels, out_channels, kernel_size=1, bias=bias),
+            nn.BatchNorm2d(out_channels))
+
+        # Remember that the stride is the same as the kernel_size, just like
+        # the max pooling layers
+        self.main_unpool1 = nn.MaxUnpool2d(kernel_size=2)
+
+        # Extension branch - 1x1 convolution, followed by a regular, dilated or
+        # asymmetric convolution, followed by another 1x1 convolution. Number
+        # of channels is doubled.
+
+        # 1x1 projection convolution with stride 1
+        self.ext_conv1 = nn.Sequential(
+            nn.Conv2d(in_channels, internal_channels, kernel_size=1, bias=bias),
+            nn.BatchNorm2d(internal_channels), activation())
+
+        # Transposed convolution
+        self.ext_tconv1 = nn.ConvTranspose2d(internal_channels, internal_channels, kernel_size=2, stride=2, bias=bias)
+        self.ext_tconv1_bnorm = nn.BatchNorm2d(internal_channels)
+        self.ext_tconv1_activation = activation()
+
+        # 1x1 expansion convolution
+        self.ext_conv2 = nn.Sequential(
+            nn.Conv2d(internal_channels, out_channels, kernel_size=1, bias=bias),
+            nn.BatchNorm2d(out_channels),
+            # activation()
+        )
+
+        self.ext_regul = nn.Dropout2d(p=dropout_prob)
+
+        # PReLU layer to apply after concatenating the branches
+        self.out_activation = activation()
+
+    def forward(self, x, max_indices, output_size):
+        # Main branch shortcut
+        main = self.main_conv1(x)
+        main = self.main_unpool1(main, max_indices, output_size=output_size)
+
+        # Extension branch
+        ext = self.ext_conv1(x)
+        ext = self.ext_tconv1(ext, output_size=output_size)
+        ext = self.ext_tconv1_bnorm(ext)
+        ext = self.ext_tconv1_activation(ext)
+        ext = self.ext_conv2(ext)
+        if self.is_dropout:
+            ext = self.ext_regul(ext)
+
+        # Add main and extension branches
+        out = main + ext
+
+        return self.out_activation(out)
+
+
+class SpatialSoftmax(nn.Module):
+    def __init__(self, temperature=1, device='cpu'):
+        super(SpatialSoftmax, self).__init__()
+
+        if temperature:
+            self.temperature = Parameter(torch.ones(1) * temperature)
+        else:
+            self.temperature = 1.
+
+    def forward(self, feature):
+        feature = feature.view(feature.shape[0], -1, feature.shape[1] * feature.shape[2])
+        softmax_attention = F.softmax(feature / self.temperature, dim=-1)
+
+        return softmax_attention
+
+
+# class LaneExistENet(nn.Module):
+#     def __init__(self, num_output, num_lane, dropout=0.1, flattened_size=4500):
+#         super().__init__()
+#         self.conv_dilated = nn.Conv2d(128, 32, kernel_size=3, stride=1, padding=4, dilation=4)
+#         self.bn = nn.BatchNorm2d(32)
+#         self.dropout = nn.Dropout2d(dropout)
+#         self.conv1x1 = nn.Conv2d(32, num_lane, kernel_size=1, stride=1)
+#         self.ssm = SpatialSoftmax()
+#         self.avgpool = nn.AvgPool2d(2, 2)
+#         self.linear1 = nn.Linear(flattened_size, 128)
+#         self.linear2 = nn.Linear(128, num_output)
+#         self.sigmoid = nn.Sigmoid()
+#
+#     def forward(self, input):
+#         # input: logits
+#         output = self.conv_dilated(input)
+#         output = self.bn(output)
+#         output = F.relu(output)
+#         output = self.ssm(output)
+#         output = self.avgpool(output)
+#         output = output.flatten(start_dim=1)
+#         output = self.linear1(output)
+#         output = F.relu(output)
+#         output = self.linear2(output)
+#         output = self.sigmoid(output)
+#         return output
+
+
+# class ENet(nn.Module):
+#     """Generate the ENet model.
+#     Keyword arguments:
+#     - num_classes (int): the number of classes to segment.
+#     - encoder_relu (bool, optional): When ``True`` ReLU is used as the
+#     activation function in the encoder blocks/layers; otherwise, PReLU
+#     is used. Default: False.
+#     - decoder_relu (bool, optional): When ``True`` ReLU is used as the
+#     activation function in the decoder blocks/layers; otherwise, PReLU
+#     is used. Default: True.
+#     """
+#
+#     def __init__(self, num_classes, encoder_relu=False, decoder_relu=True, dropout_1=0.01, dropout_2=0.1):
+#         super().__init__()
+#
+#         self.initial_block = InitialBlock(3, 16, relu=encoder_relu)
+#
+#         # Stage 1 - Encoder
+#         self.downsample1_0 = DownsamplingBottleneck(16, 64, return_indices=True, dropout_prob=dropout_1,
+#                                                     relu=encoder_relu)
+#         self.regular1_1 = RegularBottleneck(64, padding=1, dropout_prob=dropout_1, relu=encoder_relu)
+#         self.regular1_2 = RegularBottleneck(64, padding=1, dropout_prob=dropout_1, relu=encoder_relu)
+#         self.regular1_3 = RegularBottleneck(64, padding=1, dropout_prob=dropout_1, relu=encoder_relu)
+#         self.regular1_4 = RegularBottleneck(64, padding=1, dropout_prob=dropout_1, relu=encoder_relu)
+#
+#         # Stage 2 - Encoder
+#         self.downsample2_0 = DownsamplingBottleneck(64, 128, return_indices=True, dropout_prob=dropout_2,
+#                                                     relu=encoder_relu)
+#         self.regular2_1 = RegularBottleneck(128, padding=1, dropout_prob=dropout_2, relu=encoder_relu)
+#         self.dilated2_2 = RegularBottleneck(128, dilation=2, padding=2, dropout_prob=dropout_2, relu=encoder_relu)
+#         self.asymmetric2_3 = RegularBottleneck(128, kernel_size=5, padding=2, asymmetric=True, dropout_prob=dropout_2,
+#                                                relu=encoder_relu)
+#         self.dilated2_4 = RegularBottleneck(128, dilation=4, padding=4, dropout_prob=dropout_2, relu=encoder_relu)
+#         self.regular2_5 = RegularBottleneck(128, padding=1, dropout_prob=dropout_2, relu=encoder_relu)
+#         self.dilated2_6 = RegularBottleneck(128, dilation=8, padding=8, dropout_prob=dropout_2, relu=encoder_relu)
+#         self.asymmetric2_7 = RegularBottleneck(128, kernel_size=5, asymmetric=True, padding=2, dropout_prob=dropout_2,
+#                                                relu=encoder_relu)
+#         self.dilated2_8 = RegularBottleneck(128, dilation=16, padding=16, dropout_prob=dropout_2, relu=encoder_relu)
+#
+#         # Stage 3 - Encoder
+#         self.regular3_0 = RegularBottleneck(128, padding=1, dropout_prob=dropout_2, relu=encoder_relu)
+#         self.dilated3_1 = RegularBottleneck(128, dilation=2, padding=2, dropout_prob=dropout_2, relu=encoder_relu)
+#         self.asymmetric3_2 = RegularBottleneck(128, kernel_size=5, padding=2, asymmetric=True, dropout_prob=dropout_2,
+#                                                relu=encoder_relu)
+#         self.dilated3_3 = RegularBottleneck(128, dilation=4, padding=4, dropout_prob=dropout_2, relu=encoder_relu)
+#         self.regular3_4 = RegularBottleneck(128, padding=1, dropout_prob=dropout_2, relu=encoder_relu)
+#         self.dilated3_5 = RegularBottleneck(128, dilation=8, padding=8, dropout_prob=dropout_2, relu=encoder_relu)
+#         self.asymmetric3_6 = RegularBottleneck(128, kernel_size=5, asymmetric=True, padding=2, dropout_prob=dropout_2,
+#                                                relu=encoder_relu)
+#         self.dilated3_7 = RegularBottleneck(128, dilation=16, padding=16, dropout_prob=dropout_2, relu=encoder_relu)
+#
+#         # Stage 4 - Decoder
+#         self.upsample4_0 = UpsamplingBottleneck(128, 64, dropout_prob=dropout_2, is_dropout=False, relu=decoder_relu)
+#         self.regular4_1 = RegularBottleneck(64, padding=1, dropout_prob=dropout_2, is_dropout=False, relu=decoder_relu)
+#         self.regular4_2 = RegularBottleneck(64, padding=1, dropout_prob=dropout_2, is_dropout=False, relu=decoder_relu)
+#
+#         # Stage 5 - Decoder
+#         self.upsample5_0 = UpsamplingBottleneck(64, 16, dropout_prob=dropout_2, is_dropout=False, relu=decoder_relu)
+#         self.regular5_1 = RegularBottleneck(16, padding=1, dropout_prob=dropout_2, is_dropout=False, relu=decoder_relu)
+#         self.transposed_conv = nn.ConvTranspose2d(16, num_classes, kernel_size=3, stride=2, padding=1, bias=False)
+#
+#     def forward(self, x):
+#         # Initial block
+#         input_size = x.size()
+#         x = self.initial_block(x)
+#
+#         # Stage 1 - Encoder
+#         stage1_input_size = x.size()
+#         x, max_indices1_0 = self.downsample1_0(x)
+#         x = self.regular1_1(x)
+#         x = self.regular1_2(x)
+#         x = self.regular1_3(x)
+#         x = self.regular1_4(x)
+#
+#         # Stage 2 - Encoder
+#         stage2_input_size = x.size()
+#         x, max_indices2_0 = self.downsample2_0(x)
+#         x = self.regular2_1(x)
+#         x = self.dilated2_2(x)
+#         x = self.asymmetric2_3(x)
+#         x = self.dilated2_4(x)
+#         x = self.regular2_5(x)
+#         x = self.dilated2_6(x)
+#         x = self.asymmetric2_7(x)
+#         x = self.dilated2_8(x)
+#
+#         # Stage 3 - Encoder
+#         x = self.regular3_0(x)
+#         x = self.dilated3_1(x)
+#         x = self.asymmetric3_2(x)
+#         x = self.dilated3_3(x)
+#         x = self.regular3_4(x)
+#         x = self.dilated3_5(x)
+#         x = self.asymmetric3_6(x)
+#         x = self.dilated3_7(x)
+#
+#         # Stage 4 - Decoder
+#         x = self.upsample4_0(x, max_indices2_0, output_size=stage2_input_size)
+#         x = self.regular4_1(x)
+#         x = self.regular4_2(x)
+#
+#         # Stage 5 - Decoder
+#         x = self.upsample5_0(x, max_indices1_0, output_size=stage1_input_size)
+#         x = self.regular5_1(x)
+#         x = self.transposed_conv(x, output_size=input_size)
+#
+#         return x
+
+
+class Encoder(nn.Module):
+
+    def __init__(self, encoder_relu=False, dropout_1=0.01, dropout_2=0.1):
+        super().__init__()
+
+        self.initial_block = InitialBlock(3, 16, relu=encoder_relu)
+
+        # Stage 1 - Encoder
+        self.downsample1_0 = DownsamplingBottleneck(16, 64, return_indices=True, dropout_prob=dropout_1,
+                                                    relu=encoder_relu)
+        self.regular1_1 = RegularBottleneck(64, padding=1, dropout_prob=dropout_1, relu=encoder_relu)
+        self.regular1_2 = RegularBottleneck(64, padding=1, dropout_prob=dropout_1, relu=encoder_relu)
+        self.regular1_3 = RegularBottleneck(64, padding=1, dropout_prob=dropout_1, relu=encoder_relu)
+        self.regular1_4 = RegularBottleneck(64, padding=1, dropout_prob=dropout_1, relu=encoder_relu)
+
+        # Stage 2 - Encoder
+        self.downsample2_0 = DownsamplingBottleneck(64, 128, return_indices=True, dropout_prob=dropout_2,
+                                                    relu=encoder_relu)
+        self.regular2_1 = RegularBottleneck(128, padding=1, dropout_prob=dropout_2, relu=encoder_relu)
+        self.dilated2_2 = RegularBottleneck(128, dilation=2, padding=2, dropout_prob=dropout_2, relu=encoder_relu)
+        self.asymmetric2_3 = RegularBottleneck(128, kernel_size=5, padding=2, asymmetric=True, dropout_prob=dropout_2,
+                                               relu=encoder_relu)
+        self.dilated2_4 = RegularBottleneck(128, dilation=4, padding=4, dropout_prob=dropout_2, relu=encoder_relu)
+        self.regular2_5 = RegularBottleneck(128, padding=1, dropout_prob=dropout_2, relu=encoder_relu)
+        self.dilated2_6 = RegularBottleneck(128, dilation=8, padding=8, dropout_prob=dropout_2, relu=encoder_relu)
+        self.asymmetric2_7 = RegularBottleneck(128, kernel_size=5, asymmetric=True, padding=2, dropout_prob=dropout_2,
+                                               relu=encoder_relu)
+        self.dilated2_8 = RegularBottleneck(128, dilation=16, padding=16, dropout_prob=dropout_2, relu=encoder_relu)
+
+        # Stage 3 - Encoder
+        self.regular3_0 = RegularBottleneck(128, padding=1, dropout_prob=dropout_2, relu=encoder_relu)
+        self.dilated3_1 = RegularBottleneck(128, dilation=2, padding=2, dropout_prob=dropout_2, relu=encoder_relu)
+        self.asymmetric3_2 = RegularBottleneck(128, kernel_size=5, padding=2, asymmetric=True, dropout_prob=dropout_2,
+                                               relu=encoder_relu)
+        self.dilated3_3 = RegularBottleneck(128, dilation=4, padding=4, dropout_prob=dropout_2, relu=encoder_relu)
+        self.regular3_4 = RegularBottleneck(128, padding=1, dropout_prob=dropout_2, relu=encoder_relu)
+        self.dilated3_5 = RegularBottleneck(128, dilation=8, padding=8, dropout_prob=dropout_2, relu=encoder_relu)
+        self.asymmetric3_6 = RegularBottleneck(128, kernel_size=5, asymmetric=True, padding=2, dropout_prob=dropout_2,
+                                               relu=encoder_relu)
+        self.dilated3_7 = RegularBottleneck(128, dilation=16, padding=16, dropout_prob=dropout_2, relu=encoder_relu)
+
+    def forward(self, x):
+        # Initial block
+        input_size = x.size()
+        x = self.initial_block(x)
+
+        # Stage 1 - Encoder
+        stage1_input_size = x.size()
+        x, max_indices1_0 = self.downsample1_0(x)
+        x = self.regular1_1(x)
+        x = self.regular1_2(x)
+        x = self.regular1_3(x)
+        x = self.regular1_4(x)
+
+        # Stage 2 - Encoder
+        stage2_input_size = x.size()
+        x, max_indices2_0 = self.downsample2_0(x)
+        x = self.regular2_1(x)
+        x = self.dilated2_2(x)
+        x = self.asymmetric2_3(x)
+        x = self.dilated2_4(x)
+        x = self.regular2_5(x)
+        x = self.dilated2_6(x)
+        x = self.asymmetric2_7(x)
+        x = self.dilated2_8(x)
+
+        # Stage 3 - Encoder
+        x = self.regular3_0(x)
+        x = self.dilated3_1(x)
+        x = self.asymmetric3_2(x)
+        x = self.dilated3_3(x)
+        x = self.regular3_4(x)
+        x = self.dilated3_5(x)
+        x = self.asymmetric3_6(x)
+        x = self.dilated3_7(x)
+
+        return x, max_indices1_0, stage1_input_size, max_indices2_0, stage2_input_size, input_size
+
+
+class Decoder(nn.Module):
+
+    def __init__(self, num_classes, decoder_relu=True, dropout_2=0.1):
+        super().__init__()
+
+        # Stage 4 - Decoder
+        self.upsample4_0 = UpsamplingBottleneck(128, 64, dropout_prob=dropout_2, is_dropout=False, relu=decoder_relu)
+        self.regular4_1 = RegularBottleneck(64, padding=1, dropout_prob=dropout_2, is_dropout=False, relu=decoder_relu)
+        self.regular4_2 = RegularBottleneck(64, padding=1, dropout_prob=dropout_2, is_dropout=False, relu=decoder_relu)
+
+        # Stage 5 - Decoder
+        self.upsample5_0 = UpsamplingBottleneck(64, 16, dropout_prob=dropout_2, is_dropout=False, relu=decoder_relu)
+        self.regular5_1 = RegularBottleneck(16, padding=1, dropout_prob=dropout_2, is_dropout=False, relu=decoder_relu)
+        self.transposed_conv = nn.ConvTranspose2d(16, num_classes, kernel_size=3, stride=2, padding=1, bias=False)
+
+    def forward(self, x, max_indices1_0, stage1_input_size, max_indices2_0, stage2_input_size, input_size):
+        # Stage 4 - Decoder
+        x = self.upsample4_0(x, max_indices2_0, output_size=stage2_input_size)
+        x = self.regular4_1(x)
+        x = self.regular4_2(x)
+
+        # Stage 5 - Decoder
+        x = self.upsample5_0(x, max_indices1_0, output_size=stage1_input_size)
+        x = self.regular5_1(x)
+        x = self.transposed_conv(x, output_size=input_size)
+
+        return x
+
+
+@MODELS.register()
+class ENet(_EncoderDecoderModel):
+    def __init__(self,
+                 num_classes,
+                 lane_classifier_cfg=None,
+                 encoder_relu=False,
+                 decoder_relu=True,
+                 dropout_1=0.01,
+                 dropout_2=0.1,
+                 encoder_only=False,
+                 pretrained_weights=None):
+        super().__init__()
+        self.encoder_conv = None
+        self.encoder = Encoder(encoder_relu=encoder_relu, dropout_1=dropout_1, dropout_2=dropout_2)
+
+        if encoder_only:
+            self.decoder = None
+            self.encoder_conv = nn.Conv2d(128, num_classes, kernel_size=1, stride=1)
+        else:
+            self.decoder = Decoder(num_classes=num_classes, decoder_relu=decoder_relu, dropout_2=dropout_2)
+
+        self.lane_classifier = MODELS.from_dict(lane_classifier_cfg)
+        self._load_encoder(pretrained_weights)
+
+    def forward(self, x):
+        out = OrderedDict()
+        x, max_indices1_0, stage1_input_size, max_indices2_0, stage2_input_size, input_size = self.encoder(x)
+        if self.encoder_conv is not None:
+            x = self.encoder_conv(x)
+
+        if self.lane_classifier is not None:
+            out['lane'] = self.lane_classifier(x)
+
+        if self.decoder is not None:
+            x = self.decoder.forward(x, max_indices1_0, stage1_input_size, max_indices2_0,
+                                     stage2_input_size, input_size)
+        out['out'] = x
+
+        return out

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/segmentation/erfnet.py

@@ -0,0 +1,144 @@
+# Copied and modified from Eromera/erfnet_pytorch,
+# cardwing/Codes-for-Lane-Detection and
+# jcdubron/scnn_pytorch
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from collections import OrderedDict
+
+from ..common_models import non_bottleneck_1d
+from ..builder import MODELS
+from ._utils import _EncoderDecoderModel
+
+
+class DownsamplerBlock(nn.Module):
+    def __init__(self, ninput, noutput):
+        super().__init__()
+        self.conv = nn.Conv2d(ninput, noutput-ninput, (3, 3), stride=2, padding=1, bias=True)
+        self.pool = nn.MaxPool2d(2, stride=2)
+        self.bn = nn.BatchNorm2d(noutput, eps=1e-3)
+
+    def forward(self, input):
+        output = torch.cat([self.conv(input), self.pool(input)], 1)
+        output = self.bn(output)
+        return F.relu(output)
+
+
+class Encoder(nn.Module):
+    def __init__(self, num_classes, dropout_1=0.03, dropout_2=0.3):
+        super().__init__()
+        self.initial_block = DownsamplerBlock(3, 16)
+
+        self.layers = nn.ModuleList()
+
+        self.layers.append(DownsamplerBlock(16, 64))
+
+        for x in range(0, 5):  # 5 times
+            self.layers.append(non_bottleneck_1d(64, dropout_1, 1))
+
+        self.layers.append(DownsamplerBlock(64, 128))
+
+        for x in range(0, 2):  # 2 times
+            self.layers.append(non_bottleneck_1d(128, dropout_2, 2))
+            self.layers.append(non_bottleneck_1d(128, dropout_2, 4))
+            self.layers.append(non_bottleneck_1d(128, dropout_2, 8))
+            self.layers.append(non_bottleneck_1d(128, dropout_2, 16))
+
+        # Only in encoder mode:
+        self.output_conv = nn.Conv2d(128, num_classes, 1, stride=1, padding=0, bias=True)
+
+    def forward(self, input, predict=False):
+        output = self.initial_block(input)
+
+        for layer in self.layers:
+            output = layer(output)
+
+        if predict:
+            output = self.output_conv(output)
+
+        return output
+
+
+class UpsamplerBlock(nn.Module):
+    def __init__(self, ninput, noutput):
+        super().__init__()
+        self.conv = nn.ConvTranspose2d(ninput, noutput, 3, stride=2, padding=1, output_padding=1, bias=True)
+        self.bn = nn.BatchNorm2d(noutput, eps=1e-3)
+
+    def forward(self, input):
+        output = self.conv(input)
+        output = self.bn(output)
+        return F.relu(output)
+
+
+class Decoder(nn.Module):
+    def __init__(self, num_classes):
+        super().__init__()
+        self.layers = nn.ModuleList()
+        self.layers.append(UpsamplerBlock(128, 64))
+        self.layers.append(non_bottleneck_1d(64, 0, 1))
+        self.layers.append(non_bottleneck_1d(64, 0, 1))
+        self.layers.append(UpsamplerBlock(64, 16))
+        self.layers.append(non_bottleneck_1d(16, 0, 1))
+        self.layers.append(non_bottleneck_1d(16, 0, 1))
+
+        self.output_conv = nn.ConvTranspose2d(16, num_classes, 2, stride=2, padding=0, output_padding=0, bias=True)
+
+    def forward(self, input):
+        output = input
+
+        for layer in self.layers:
+            output = layer(output)
+
+        output = self.output_conv(output)
+
+        return output
+
+
+# ERFNet
+@MODELS.register()
+class ERFNet(_EncoderDecoderModel):
+    def __init__(self,
+                 num_classes,
+                 lane_classifier_cfg=None,
+                 spatial_conv_cfg=None,
+                 dropout_1=0.03,
+                 dropout_2=0.3,
+                 pretrained_weights=None):
+        super().__init__()
+        self.encoder = Encoder(num_classes=num_classes, dropout_1=dropout_1, dropout_2=dropout_2)
+        self.decoder = Decoder(num_classes)
+        self.spatial_conv = MODELS.from_dict(spatial_conv_cfg)
+        self.lane_classifier = MODELS.from_dict(lane_classifier_cfg)
+        self._load_encoder(pretrained_weights)
+
+    def _load_encoder(self, pretrained_weights):
+        if pretrained_weights is not None:  # Load ImageNet pre-trained weights
+            try:
+                saved_weights = torch.load(pretrained_weights)['state_dict']
+            except FileNotFoundError:
+                raise FileNotFoundError('pretrained_weights is not there! '
+                                        'Please set pretrained_weights=None if you are only testing.')
+            original_weights = self.state_dict()
+            for key in saved_weights.keys():
+                my_key = key.replace('module.features.', '')
+                if my_key in original_weights.keys():
+                    original_weights[my_key] = saved_weights[key]
+            self.load_state_dict(original_weights)
+        else:
+            print('No ImageNet pre-training.')
+
+    def forward(self, x, only_encode=False):
+        out = OrderedDict()
+        if only_encode:
+            return self.encoder.forward(x, predict=True)
+        else:
+            output = self.encoder(x)    # predict=False by default
+            if self.spatial_conv is not None:
+                output = self.spatial_conv(output)
+            out['out'] = self.decoder.forward(output)
+
+            if self.lane_classifier is not None:
+                out['lane'] = self.lane_classifier(output)
+            return out

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/segmentation/fcn.py

@@ -0,0 +1,18 @@
+from torch import nn
+
+from ..builder import MODELS
+
+
+@MODELS.register()
+class FCNHead(nn.Sequential):
+    def __init__(self, in_channels, num_classes):
+        inter_channels = in_channels // 4
+        layers = [
+            nn.Conv2d(in_channels, inter_channels, 3, padding=1, bias=False),
+            nn.BatchNorm2d(inter_channels),
+            nn.ReLU(),
+            nn.Dropout(0.1),
+            nn.Conv2d(inter_channels, num_classes, 1)
+        ]
+
+        super(FCNHead, self).__init__(*layers)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/segmentation/segmentation.py

@@ -0,0 +1,59 @@
+from ..utils import load_state_dict_from_url
+from ._utils import _SimpleSegmentationModel
+from ..builder import MODELS
+
+model_urls = {
+    'fcn_resnet50_coco': None,
+    'fcn_resnet101_coco': 'https://download.pytorch.org/models/fcn_resnet101_coco-7ecb50ca.pth',
+    'deeplabv3_resnet50_coco': None,
+    'deeplabv3_resnet101_coco': 'https://download.pytorch.org/models/deeplabv3_resnet101_coco-586e9e4e.pth',
+    'deeplabv2_resnet101_coco': None,
+    'deeplabv1_resnet18_coco': None,
+    'deeplabv1_resnet34_coco': None,
+    'deeplabv1_resnet50_coco': None,
+    'deeplabv1_resnet101_coco': None
+}
+
+
+def _build_segmentation_net(backbone_cfg, classifier_cfg,
+                            aux_classifier_cfg=None, lane_classifier_cfg=None, reducer_cfg=None, spatial_conv_cfg=None):
+    if aux_classifier_cfg is not None:  # Fixed for COCO pre-training
+        backbone_cfg['return_layer'] = {
+            'layer3': 'aux',
+            'layer4': 'out'
+        }
+    backbone = MODELS.from_dict(backbone_cfg)
+    classifier = MODELS.from_dict(classifier_cfg)
+    aux_classifier = MODELS.from_dict(aux_classifier_cfg)
+    lane_classifier = MODELS.from_dict(lane_classifier_cfg)
+    channel_reducer = MODELS.from_dict(reducer_cfg)
+    scnn_layer = MODELS.from_dict(spatial_conv_cfg)
+
+    model = _SimpleSegmentationModel(
+        backbone, classifier, aux_classifier,
+        lane_classifier=lane_classifier, channel_reducer=channel_reducer, scnn_layer=scnn_layer)
+
+    return model
+
+
+@MODELS.register()
+def standard_segmentation_model(backbone_cfg, classifier_cfg,
+                                progress=True, pretrained=False, arch_type=None, backbone=None,
+                                aux_classifier_cfg=None,
+                                lane_classifier_cfg=None, reducer_cfg=None, spatial_conv_cfg=None):
+    if pretrained:
+        assert 'aux_classifier_cfg' is not None
+        assert arch_type is not None
+        assert backbone is not None
+    model = _build_segmentation_net(backbone_cfg, classifier_cfg,
+                                    aux_classifier_cfg, lane_classifier_cfg, reducer_cfg, spatial_conv_cfg)
+    if pretrained:
+        arch = arch_type + '_' + backbone + '_coco'
+        model_url = model_urls[arch]
+        if model_url is None:
+            raise NotImplementedError('COCO pretrained {} is not supported as of now'.format(arch))
+        else:
+            state_dict = load_state_dict_from_url(model_url, progress=progress)
+            model.load_state_dict(state_dict)
+
+    return model

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/segmentation/simple_seg_head.py

@@ -0,0 +1,63 @@
+# Simplified from MichaelFan01/STDC-Seg
+import torch.nn as nn
+from torch.nn import BatchNorm2d
+
+from ..builder import MODELS
+
+
+class ConvBNReLU(nn.Module):
+    def __init__(self, in_chan, out_chan, ks=3, stride=1, padding=1):
+        super(ConvBNReLU, self).__init__()
+        self.conv = nn.Conv2d(in_chan,
+                              out_chan,
+                              kernel_size = ks,
+                              stride = stride,
+                              padding = padding,
+                              bias = False)
+        self.bn = BatchNorm2d(out_chan)
+        self.relu = nn.ReLU()
+        self.init_weight()
+
+    def forward(self, x):
+        x = self.conv(x)
+        x = self.bn(x)
+        x = self.relu(x)
+        return x
+
+    def init_weight(self):
+        for ly in self.children():
+            if isinstance(ly, nn.Conv2d):
+                nn.init.kaiming_normal_(ly.weight, a=1)
+                if not ly.bias is None: nn.init.constant_(ly.bias, 0)
+
+
+@MODELS.register()
+class SimpleSegHead(nn.Module):
+    # Seg head (Generalized FCN head without dropout)
+    def __init__(self, in_channels, mid_channels, num_classes):
+        super().__init__()
+        self.conv = ConvBNReLU(in_channels, mid_channels, ks=3, stride=1, padding=1)
+        self.conv_out = nn.Conv2d(mid_channels, num_classes, kernel_size=1, bias=False)
+        self.init_weight()
+
+    def forward(self, x):
+        x = self.conv(x)
+        x = self.conv_out(x)
+        return x
+
+    def init_weight(self):
+        for ly in self.children():
+            if isinstance(ly, nn.Conv2d):
+                nn.init.kaiming_normal_(ly.weight, a=1)
+                if not ly.bias is None: nn.init.constant_(ly.bias, 0)
+
+    def get_params(self):
+        wd_params, nowd_params = [], []
+        for _, module in self.named_modules():
+            if isinstance(module, (nn.Linear, nn.Conv2d)):
+                wd_params.append(module.weight)
+                if not module.bias is None:
+                    nowd_params.append(module.bias)
+            elif isinstance(module, BatchNorm2d):
+                nowd_params += list(module.parameters())
+        return wd_params, nowd_params

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/swin.py

@@ -0,0 +1,626 @@
+# --------------------------------------------------------
+# Swin Transformer
+# Copyright (c) 2021 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+# Written by Ze Liu, Yutong Lin, Yixuan Wei
+# --------------------------------------------------------
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.utils.checkpoint as checkpoint
+import numpy as np
+from timm.models.layers import DropPath, to_2tuple, trunc_normal_
+from .builder import MODELS
+
+
+class Mlp(nn.Module):
+    """ Multilayer perceptron."""
+
+    def __init__(self, in_features, hidden_features=None, out_features=None, act_fn=F.gelu, drop=0.):
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        self.fc1 = nn.Linear(in_features, hidden_features)
+        self.act = act_fn
+        self.fc2 = nn.Linear(hidden_features, out_features)
+        self.drop = nn.Dropout(drop)
+
+    def forward(self, x):
+        x = self.fc1(x)
+        x = self.act(x)
+        x = self.drop(x)
+        x = self.fc2(x)
+        x = self.drop(x)
+        return x
+
+
+def window_partition(x, window_size):
+    """
+    Args:
+        x: (B, H, W, C)
+        window_size (int): window size
+
+    Returns:
+        windows: (num_windows*B, window_size, window_size, C)
+    """
+    B, H, W, C = x.shape
+    x = x.view(B, H // window_size, window_size, W // window_size, window_size, C)
+    windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C)
+    return windows
+
+
+def window_reverse(windows, window_size, H, W):
+    """
+    Args:
+        windows: (num_windows*B, window_size, window_size, C)
+        window_size (int): Window size
+        H (int): Height of image
+        W (int): Width of image
+
+    Returns:
+        x: (B, H, W, C)
+    """
+    B = int(windows.shape[0] / (H * W / float(window_size) / window_size))
+    x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1)
+    x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1)
+    return x
+
+
+class WindowAttention(nn.Module):
+    """ Window based multi-head self attention (W-MSA) module with relative position bias.
+    It supports both of shifted and non-shifted window.
+
+    Args:
+        dim (int): Number of input channels.
+        window_size (tuple[int]): The height and width of the window.
+        num_heads (int): Number of attention heads.
+        qkv_bias (bool, optional):  If True, add a learnable bias to query, key, value. Default: True
+        qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set
+        attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0
+        proj_drop (float, optional): Dropout ratio of output. Default: 0.0
+    """
+
+    def __init__(self, dim, window_size, num_heads, qkv_bias=True, qk_scale=None, attn_drop=0., proj_drop=0.):
+
+        super().__init__()
+        self.dim = dim
+        self.window_size = window_size  # Wh, Ww
+        self.num_heads = num_heads
+        head_dim = dim // num_heads
+        self.scale = qk_scale or head_dim ** -0.5
+
+        # define a parameter table of relative position bias
+        self.relative_position_bias_table = nn.Parameter(
+            torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), num_heads))  # 2*Wh-1 * 2*Ww-1, nH
+
+        # get pair-wise relative position index for each token inside the window
+        coords_h = torch.arange(self.window_size[0])
+        coords_w = torch.arange(self.window_size[1])
+        coords = torch.stack(torch.meshgrid([coords_h, coords_w]))  # 2, Wh, Ww
+        coords_flatten = torch.flatten(coords, 1)  # 2, Wh*Ww
+        relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :]  # 2, Wh*Ww, Wh*Ww
+        relative_coords = relative_coords.permute(1, 2, 0).contiguous()  # Wh*Ww, Wh*Ww, 2
+        relative_coords[:, :, 0] += self.window_size[0] - 1  # shift to start from 0
+        relative_coords[:, :, 1] += self.window_size[1] - 1
+        relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1
+        relative_position_index = relative_coords.sum(-1)  # Wh*Ww, Wh*Ww
+        self.register_buffer("relative_position_index", relative_position_index)
+
+        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
+        self.attn_drop = nn.Dropout(attn_drop)
+        self.proj = nn.Linear(dim, dim)
+        self.proj_drop = nn.Dropout(proj_drop)
+
+        trunc_normal_(self.relative_position_bias_table, std=.02)
+        self.softmax = nn.Softmax(dim=-1)
+
+    def forward(self, x, mask=None):
+        """ Forward function.
+
+        Args:
+            x: input features with shape of (num_windows*B, N, C)
+            mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None
+        """
+        B_, N, C = x.shape
+        qkv = self.qkv(x).reshape(B_, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
+        q, k, v = qkv[0], qkv[1], qkv[2]  # make torchscript happy (cannot use tensor as tuple)
+
+        q = q * self.scale
+        attn = (q @ k.transpose(-2, -1))
+
+        relative_position_bias = self.relative_position_bias_table[self.relative_position_index.view(-1)].view(
+            self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1)  # Wh*Ww,Wh*Ww,nH
+        relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous()  # nH, Wh*Ww, Wh*Ww
+        attn = attn + relative_position_bias.unsqueeze(0)
+
+        if mask is not None:
+            nW = mask.shape[0]
+            attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0)
+            attn = attn.view(-1, self.num_heads, N, N)
+            attn = self.softmax(attn)
+        else:
+            attn = self.softmax(attn)
+
+        attn = self.attn_drop(attn)
+
+        x = (attn @ v).transpose(1, 2).reshape(B_, N, C)
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x
+
+
+class SwinTransformerBlock(nn.Module):
+    """ Swin Transformer Block.
+
+    Args:
+        dim (int): Number of input channels.
+        num_heads (int): Number of attention heads.
+        window_size (int): Window size.
+        shift_size (int): Shift size for SW-MSA.
+        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
+        qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True
+        qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set.
+        drop (float, optional): Dropout rate. Default: 0.0
+        attn_drop (float, optional): Attention dropout rate. Default: 0.0
+        drop_path (float, optional): Stochastic depth rate. Default: 0.0
+        act_fn (nn.Module, optional): Activation layer. Default: F.gelu
+        norm_layer (nn.Module, optional): Normalization layer.  Default: nn.LayerNorm
+    """
+
+    def __init__(self, dim, num_heads, window_size=7, shift_size=0,
+                 mlp_ratio=4., qkv_bias=True, qk_scale=None, drop=0., attn_drop=0., drop_path=0.,
+                 act_fn=F.gelu, norm_layer=nn.LayerNorm):
+        super().__init__()
+        self.dim = dim
+        self.num_heads = num_heads
+        self.window_size = window_size
+        self.shift_size = shift_size
+        self.mlp_ratio = mlp_ratio
+        assert 0 <= self.shift_size < self.window_size, "shift_size must in 0-window_size"
+
+        self.norm1 = norm_layer(dim)
+        self.attn = WindowAttention(
+            dim, window_size=to_2tuple(self.window_size), num_heads=num_heads,
+            qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop)
+
+        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
+        self.norm2 = norm_layer(dim)
+        mlp_hidden_dim = int(dim * mlp_ratio)
+        self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_fn=act_fn, drop=drop)
+
+        self.H = None
+        self.W = None
+
+    def forward(self, x, mask_matrix):
+        """ Forward function.
+
+        Args:
+            x: Input feature, tensor size (B, H*W, C).
+            H, W: Spatial resolution of the input feature.
+            mask_matrix: Attention mask for cyclic shift.
+        """
+        B, L, C = x.shape
+        H, W = self.H, self.W
+        assert L == H * W, "input feature has wrong size"
+
+        shortcut = x
+        x = self.norm1(x)
+        x = x.view(B, H, W, C)
+
+        # pad feature maps to multiples of window size
+        pad_l = pad_t = 0
+        pad_r = (self.window_size - W % self.window_size) % self.window_size
+        pad_b = (self.window_size - H % self.window_size) % self.window_size
+        x = F.pad(x, (0, 0, pad_l, pad_r, pad_t, pad_b))
+        _, Hp, Wp, _ = x.shape
+
+        # cyclic shift
+        if self.shift_size > 0:
+            shifted_x = torch.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2))
+            attn_mask = mask_matrix
+        else:
+            shifted_x = x
+            attn_mask = None
+
+        # partition windows
+        x_windows = window_partition(shifted_x, self.window_size)  # nW*B, window_size, window_size, C
+        x_windows = x_windows.view(-1, self.window_size * self.window_size, C)  # nW*B, window_size*window_size, C
+
+        # W-MSA/SW-MSA
+        attn_windows = self.attn(x_windows, mask=attn_mask)  # nW*B, window_size*window_size, C
+
+        # merge windows
+        attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C)
+        shifted_x = window_reverse(attn_windows, self.window_size, Hp, Wp)  # B H' W' C
+
+        # reverse cyclic shift
+        if self.shift_size > 0:
+            x = torch.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2))
+        else:
+            x = shifted_x
+
+        if pad_r > 0 or pad_b > 0:
+            x = x[:, :H, :W, :].contiguous()
+
+        x = x.view(B, H * W, C)
+
+        # FFN
+        x = shortcut + self.drop_path(x)
+        x = x + self.drop_path(self.mlp(self.norm2(x)))
+
+        return x
+
+
+class PatchMerging(nn.Module):
+    """ Patch Merging Layer
+
+    Args:
+        dim (int): Number of input channels.
+        norm_layer (nn.Module, optional): Normalization layer.  Default: nn.LayerNorm
+    """
+    def __init__(self, dim, norm_layer=nn.LayerNorm):
+        super().__init__()
+        self.dim = dim
+        self.reduction = nn.Linear(4 * dim, 2 * dim, bias=False)
+        self.norm = norm_layer(4 * dim)
+
+    def forward(self, x, H, W):
+        """ Forward function.
+
+        Args:
+            x: Input feature, tensor size (B, H*W, C).
+            H, W: Spatial resolution of the input feature.
+        """
+        B, L, C = x.shape
+        assert L == H * W, "input feature has wrong size"
+
+        x = x.view(B, H, W, C)
+
+        # padding
+        pad_input = (H % 2 == 1) or (W % 2 == 1)
+        if pad_input:
+            x = F.pad(x, (0, 0, 0, W % 2, 0, H % 2))
+
+        x0 = x[:, 0::2, 0::2, :]  # B H/2 W/2 C
+        x1 = x[:, 1::2, 0::2, :]  # B H/2 W/2 C
+        x2 = x[:, 0::2, 1::2, :]  # B H/2 W/2 C
+        x3 = x[:, 1::2, 1::2, :]  # B H/2 W/2 C
+        x = torch.cat([x0, x1, x2, x3], -1)  # B H/2 W/2 4*C
+        x = x.view(B, -1, 4 * C)  # B H/2*W/2 4*C
+
+        x = self.norm(x)
+        x = self.reduction(x)
+
+        return x
+
+
+class BasicLayer(nn.Module):
+    """ A basic Swin Transformer layer for one stage.
+
+    Args:
+        dim (int): Number of feature channels
+        depth (int): Depths of this stage.
+        num_heads (int): Number of attention head.
+        window_size (int): Local window size. Default: 7.
+        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4.
+        qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True
+        qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set.
+        drop (float, optional): Dropout rate. Default: 0.0
+        attn_drop (float, optional): Attention dropout rate. Default: 0.0
+        drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0
+        norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm
+        downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None
+        use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False.
+    """
+
+    def __init__(self,
+                 dim,
+                 depth,
+                 num_heads,
+                 window_size=7,
+                 mlp_ratio=4.,
+                 qkv_bias=True,
+                 qk_scale=None,
+                 drop=0.,
+                 attn_drop=0.,
+                 drop_path=0.,
+                 norm_layer=nn.LayerNorm,
+                 downsample=None,
+                 use_checkpoint=False):
+        super().__init__()
+        self.window_size = window_size
+        self.shift_size = window_size // 2
+        self.depth = depth
+        self.use_checkpoint = use_checkpoint
+
+        # build blocks
+        self.blocks = nn.ModuleList([
+            SwinTransformerBlock(
+                dim=dim,
+                num_heads=num_heads,
+                window_size=window_size,
+                shift_size=0 if (i % 2 == 0) else window_size // 2,
+                mlp_ratio=mlp_ratio,
+                qkv_bias=qkv_bias,
+                qk_scale=qk_scale,
+                drop=drop,
+                attn_drop=attn_drop,
+                drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path,
+                norm_layer=norm_layer)
+            for i in range(depth)])
+
+        # patch merging layer
+        if downsample is not None:
+            self.downsample = downsample(dim=dim, norm_layer=norm_layer)
+        else:
+            self.downsample = None
+
+    def forward(self, x, H, W):
+        """ Forward function.
+
+        Args:
+            x: Input feature, tensor size (B, H*W, C).
+            H, W: Spatial resolution of the input feature.
+        """
+
+        # calculate attention mask for SW-MSA
+        Hp = int(np.ceil(float(H) / float(self.window_size))) * self.window_size
+        Wp = int(np.ceil(float(W) / float(self.window_size))) * self.window_size
+        img_mask = torch.zeros((1, Hp, Wp, 1), device=x.device)  # 1 Hp Wp 1
+        h_slices = (slice(0, -self.window_size),
+                    slice(-self.window_size, -self.shift_size),
+                    slice(-self.shift_size, None))
+        w_slices = (slice(0, -self.window_size),
+                    slice(-self.window_size, -self.shift_size),
+                    slice(-self.shift_size, None))
+        cnt = 0
+        for h in h_slices:
+            for w in w_slices:
+                img_mask[:, h, w, :] = cnt
+                cnt += 1
+
+        mask_windows = window_partition(img_mask, self.window_size)  # nW, window_size, window_size, 1
+        mask_windows = mask_windows.view(-1, self.window_size * self.window_size)
+        attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2)
+        attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill(attn_mask == 0, float(0.0))
+
+        for blk in self.blocks:
+            blk.H, blk.W = H, W
+            if self.use_checkpoint:
+                x = checkpoint.checkpoint(blk, x, attn_mask)
+            else:
+                x = blk(x, attn_mask)
+        if self.downsample is not None:
+            x_down = self.downsample(x, H, W)
+            Wh, Ww = (H + 1) // 2, (W + 1) // 2
+            return x, H, W, x_down, Wh, Ww
+        else:
+            return x, H, W, x, H, W
+
+
+class PatchEmbed(nn.Module):
+    """ Image to Patch Embedding
+
+    Args:
+        patch_size (int): Patch token size. Default: 4.
+        in_chans (int): Number of input image channels. Default: 3.
+        embed_dim (int): Number of linear projection output channels. Default: 96.
+        norm_layer (nn.Module, optional): Normalization layer. Default: None
+    """
+
+    def __init__(self, patch_size=4, in_chans=3, embed_dim=96, norm_layer=None):
+        super().__init__()
+        patch_size = to_2tuple(patch_size)
+        self.patch_size = patch_size
+
+        self.in_chans = in_chans
+        self.embed_dim = embed_dim
+
+        self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)
+        if norm_layer is not None:
+            self.norm = norm_layer(embed_dim)
+        else:
+            self.norm = None
+
+    def forward(self, x):
+        """Forward function."""
+        # padding
+        _, _, H, W = x.size()
+        if W % self.patch_size[1] != 0:
+            x = F.pad(x, (0, self.patch_size[1] - W % self.patch_size[1]))
+        if H % self.patch_size[0] != 0:
+            x = F.pad(x, (0, 0, 0, self.patch_size[0] - H % self.patch_size[0]))
+
+        x = self.proj(x)  # B C Wh Ww
+        if self.norm is not None:
+            Wh, Ww = x.size(2), x.size(3)
+            x = x.flatten(2).transpose(1, 2)
+            x = self.norm(x)
+            x = x.transpose(1, 2).view(-1, self.embed_dim, Wh, Ww)
+
+        return x
+
+
+@MODELS.register()
+class SwinTransformer(nn.Module):
+    """ Swin Transformer backbone.
+        A PyTorch impl of : `Swin Transformer: Hierarchical Vision Transformer using Shifted Windows`  -
+          https://arxiv.org/pdf/2103.14030
+
+    Args:
+        pretrain_img_size (int): Input image size for training the pretrained model,
+            used in absolute postion embedding. Default 224.
+        patch_size (int | tuple(int)): Patch size. Default: 4.
+        in_chans (int): Number of input image channels. Default: 3.
+        embed_dim (int): Number of linear projection output channels. Default: 96.
+        depths (tuple[int]): Depths of each Swin Transformer stage.
+        num_heads (tuple[int]): Number of attention head of each stage.
+        window_size (int): Window size. Default: 7.
+        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4.
+        qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True
+        qk_scale (float): Override default qk scale of head_dim ** -0.5 if set.
+        drop_rate (float): Dropout rate.
+        attn_drop_rate (float): Attention dropout rate. Default: 0.
+        drop_path_rate (float): Stochastic depth rate. Default: 0.2.
+        norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm.
+        ape (bool): If True, add absolute position embedding to the patch embedding. Default: False.
+        patch_norm (bool): If True, add normalization after patch embedding. Default: True.
+        out_indices (Sequence[int]): Output from which stages.
+        frozen_stages (int): Stages to be frozen (stop grad and set eval mode).
+            -1 means not freezing any parameters.
+        use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False.
+    """
+
+    def __init__(self,
+                 pretrain_img_size=224,
+                 patch_size=4,
+                 in_chans=3,
+                 embed_dim=96,
+                 depths=[2, 2, 6, 2],
+                 num_heads=[3, 6, 12, 24],
+                 window_size=7,
+                 mlp_ratio=4.,
+                 qkv_bias=True,
+                 qk_scale=None,
+                 drop_rate=0.,
+                 attn_drop_rate=0.,
+                 drop_path_rate=0.2,
+                 norm_layer=nn.LayerNorm,
+                 ape=False,
+                 patch_norm=True,
+                 out_indices=(0, 1, 2, 3),
+                 frozen_stages=-1,
+                 use_checkpoint=False,
+                 pretrained=None,
+                 chosen_stages=-1):
+        super().__init__()
+
+        self.pretrain_img_size = pretrain_img_size
+        self.num_layers = len(depths)
+        self.embed_dim = embed_dim
+        self.ape = ape
+        self.patch_norm = patch_norm
+        self.out_indices = out_indices
+        self.frozen_stages = frozen_stages
+        self.chosen_stages = chosen_stages
+
+        # split image into non-overlapping patches
+        self.patch_embed = PatchEmbed(
+            patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim,
+            norm_layer=norm_layer if self.patch_norm else None)
+
+        # absolute position embedding
+        if self.ape:
+            pretrain_img_size = to_2tuple(pretrain_img_size)
+            patch_size = to_2tuple(patch_size)
+            patches_resolution = [pretrain_img_size[0] // patch_size[0], pretrain_img_size[1] // patch_size[1]]
+
+            self.absolute_pos_embed = nn.Parameter(torch.zeros(1, embed_dim, patches_resolution[0], patches_resolution[1]))
+            trunc_normal_(self.absolute_pos_embed, std=.02)
+
+        self.pos_drop = nn.Dropout(p=drop_rate)
+
+        # stochastic depth
+        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))]  # stochastic depth decay rule
+
+        # build layers
+        self.layers = nn.ModuleList()
+        for i_layer in range(self.num_layers):
+            layer = BasicLayer(
+                dim=int(embed_dim * 2 ** i_layer),
+                depth=depths[i_layer],
+                num_heads=num_heads[i_layer],
+                window_size=window_size,
+                mlp_ratio=mlp_ratio,
+                qkv_bias=qkv_bias,
+                qk_scale=qk_scale,
+                drop=drop_rate,
+                attn_drop=attn_drop_rate,
+                drop_path=dpr[sum(depths[:i_layer]):sum(depths[:i_layer + 1])],
+                norm_layer=norm_layer,
+                downsample=PatchMerging if (i_layer < self.num_layers - 1) else None,
+                use_checkpoint=use_checkpoint)
+            self.layers.append(layer)
+
+        num_features = [int(embed_dim * 2 ** i) for i in range(self.num_layers)]
+        self.num_features = num_features
+
+        # add a norm layer for each output
+        for i_layer in out_indices:
+            layer = norm_layer(num_features[i_layer])
+            layer_name = f'norm{i_layer}'
+            self.add_module(layer_name, layer)
+
+        # initialize
+        if pretrained is None:
+            self.weight_initialization()
+        else:
+            assert isinstance(pretrained, str), 'pretrained must be a str'
+            self.load_checkpoint(pretrained=pretrained)
+
+        self._freeze_stages()
+
+    def _freeze_stages(self):
+        if self.frozen_stages >= 0:
+            self.patch_embed.eval()
+            for param in self.patch_embed.parameters():
+                param.requires_grad = False
+
+        if self.frozen_stages >= 1 and self.ape:
+            self.absolute_pos_embed.requires_grad = False
+
+        if self.frozen_stages >= 2:
+            self.pos_drop.eval()
+            for i in range(0, self.frozen_stages - 1):
+                m = self.layers[i]
+                m.eval()
+                for param in m.parameters():
+                    param.requires_grad = False
+
+    def weight_initialization(self):
+        for m in self.modules():
+            if isinstance(m, nn.Linear):
+                trunc_normal_(m.weight, std=.02)
+                if isinstance(m, nn.Linear) and m.bias is not None:
+                    nn.init.constant_(m.bias, 0)
+            elif isinstance(m, nn.LayerNorm):
+                nn.init.constant_(m.bias, 0)
+                nn.init.constant_(m.weight, 1.0)
+
+    def load_checkpoint(self, pretrained, strict=False):
+        state_dict = torch.load(pretrained)['model']
+        self.load_state_dict(state_dict, strict=strict)
+
+    def forward(self, x):
+        """Forward function."""
+        x = self.patch_embed(x)
+        Wh, Ww = x.size(2), x.size(3)
+        if self.ape:
+            # interpolate the position embedding to the corresponding size
+            absolute_pos_embed = F.interpolate(self.absolute_pos_embed, size=(Wh, Ww), mode='bicubic')
+            x = (x + absolute_pos_embed).flatten(2).transpose(1, 2)  # B Wh*Ww C
+        else:
+            x = x.flatten(2).transpose(1, 2)
+        x = self.pos_drop(x)
+
+        outs = []
+        for i in range(self.num_layers):
+            layer = self.layers[i]
+            x_out, H, W, x, Wh, Ww = layer(x, Wh, Ww)
+
+            if i in self.out_indices:
+                norm_layer = getattr(self, f'norm{i}')
+                x_out = norm_layer(x_out)
+
+                out = x_out.view(-1, H, W, self.num_features[i]).permute(0, 3, 1, 2).contiguous()
+                outs.append(out)
+        if self.chosen_stages == -1:
+            return tuple(outs)
+        else:
+            return tuple(outs)[self.chosen_stages]
+
+    def train(self, mode=True):
+        """Convert the model into training mode while keep layers freezed."""
+        super(SwinTransformer, self).train(mode)
+        self._freeze_stages()

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/transformer/__init__.py

@@ -0,0 +1,2 @@
+from .position_encoding import build_position_encoding
+from .transformer import Transformer, build_transformer

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/transformer/position_encoding.py

@@ -0,0 +1,90 @@
+# Copied and modified from facebookresearch/detr and liuruijin17/LSTR
+# mainly got rid of special types and argparse
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+# Positional encodings for transformers
+
+import math
+import torch
+from torch import nn
+
+from ..builder import MODELS
+
+
+@MODELS.register()
+class PositionEmbeddingSine(nn.Module):
+    """
+    This is a more standard version of the position embedding, very similar to the one
+    used by the Attention is all you need paper, generalized to work on images.
+    """
+    def __init__(self, num_pos_feats=64, temperature=10000, normalize=False, scale=None):
+        super().__init__()
+        self.num_pos_feats = num_pos_feats
+        self.temperature = temperature
+        self.normalize = normalize
+        if scale is not None and normalize is False:
+            raise ValueError("normalize should be True if scale is passed")
+        if scale is None:
+            scale = 2 * math.pi
+        self.scale = scale
+
+    def forward(self, x, mask):
+        assert mask is not None
+        not_mask = ~mask
+        y_embed = not_mask.cumsum(1, dtype=torch.float32)
+        x_embed = not_mask.cumsum(2, dtype=torch.float32)
+        if self.normalize:
+            eps = 1e-6
+            y_embed = y_embed / (y_embed[:, -1:, :] + eps) * self.scale
+            x_embed = x_embed / (x_embed[:, :, -1:] + eps) * self.scale
+
+        dim_t = torch.arange(self.num_pos_feats, dtype=torch.float32, device=x.device)
+        dim_t = self.temperature ** (2 * (dim_t // 2) / self.num_pos_feats)
+
+        pos_x = x_embed[:, :, :, None] / dim_t
+        pos_y = y_embed[:, :, :, None] / dim_t
+        pos_x = torch.stack((pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4).flatten(3)
+        pos_y = torch.stack((pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4).flatten(3)
+        pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2)
+
+        return pos
+
+
+@MODELS.register()
+class PositionEmbeddingLearned(nn.Module):
+    """
+    Absolute pos embedding, learned.
+    """
+    def __init__(self, num_pos_feats=256):
+        super().__init__()
+        self.row_embed = nn.Embedding(50, num_pos_feats)
+        self.col_embed = nn.Embedding(50, num_pos_feats)
+        self.reset_parameters()
+
+    def reset_parameters(self):
+        nn.init.uniform_(self.row_embed.weight)
+        nn.init.uniform_(self.col_embed.weight)
+
+    def forward(self, x):
+        h, w = x.shape[-2:]
+        i = torch.arange(w, device=x.device)
+        j = torch.arange(h, device=x.device)
+        x_emb = self.col_embed(i)
+        y_emb = self.row_embed(j)
+        pos = torch.cat([
+            x_emb.unsqueeze(0).repeat(h, 1, 1),
+            y_emb.unsqueeze(1).repeat(1, w, 1),
+        ], dim=-1).permute(2, 0, 1).unsqueeze(0).repeat(x.shape[0], 1, 1, 1)
+
+        return pos
+
+
+def build_position_encoding(hidden_dim, position_embedding):
+    N_steps = hidden_dim // 2
+    if position_embedding in ('v2', 'sine'):
+        position_embedding = PositionEmbeddingSine(N_steps, normalize=True)
+    elif position_embedding in ('v3', 'learned'):
+        position_embedding = PositionEmbeddingLearned(N_steps)
+    else:
+        raise ValueError(f"not supported {position_embedding}")
+
+    return position_embedding

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/transformer/transformer.py

@@ -0,0 +1,312 @@
+# Copied and modified from facebookresearch/detr and liuruijin17/LSTR
+# mainly got rid of special types and argparse, added comments
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+"""
+DETR Transformer class.
+Copy-paste from torch.nn.Transformer with modifications:
+    * positional encodings are passed in MHattention
+    * extra LN at the end of encoder is removed
+    * decoder returns a stack of activations from all decoding layers
+"""
+import copy
+from typing import Optional
+
+import torch
+import torch.nn.functional as F
+from torch import nn, Tensor
+
+from ..builder import MODELS
+
+
+@MODELS.register()
+class Transformer(nn.Module):
+    def __init__(self, d_model=512, nhead=8, num_encoder_layers=6,
+                 num_decoder_layers=6, dim_feedforward=2048, dropout=0.1,
+                 activation="relu", normalize_before=False,
+                 return_intermediate_dec=False):
+        super().__init__()
+
+        encoder_layer = TransformerEncoderLayer(d_model, nhead, dim_feedforward,
+                                                dropout, activation, normalize_before)
+        encoder_norm = nn.LayerNorm(d_model) if normalize_before else None
+        self.encoder = TransformerEncoder(encoder_layer, num_encoder_layers, encoder_norm)
+
+        decoder_layer = TransformerDecoderLayer(d_model, nhead, dim_feedforward,
+                                                dropout, activation, normalize_before)
+        decoder_norm = nn.LayerNorm(d_model)
+        self.decoder = TransformerDecoder(decoder_layer, num_decoder_layers, decoder_norm,
+                                          return_intermediate=return_intermediate_dec)
+
+        self._reset_parameters()
+
+        self.d_model = d_model
+        self.nhead = nhead
+
+    def _reset_parameters(self):
+        for p in self.parameters():
+            if p.dim() > 1:
+                nn.init.xavier_uniform_(p)
+
+    def forward(self, src, mask, query_embed, pos_embed):
+        # flatten NxCxHxW to HWxNxC (a sequence of length HW)
+        bs, c, h, w = src.shape
+        src = src.flatten(2).permute(2, 0, 1)
+        pos_embed = pos_embed.flatten(2).permute(2, 0, 1)
+        query_embed = query_embed.unsqueeze(1).repeat(1, bs, 1)
+        if mask is not None:
+            mask = mask.flatten(1)
+
+        tgt = torch.zeros_like(query_embed)
+        memory = self.encoder(src, src_key_padding_mask=mask, pos=pos_embed)
+        hs = self.decoder(tgt, memory, memory_key_padding_mask=mask,
+                          pos=pos_embed, query_pos=query_embed)
+
+        return hs.transpose(1, 2), memory.permute(1, 2, 0).view(bs, c, h, w)
+
+
+class TransformerEncoder(nn.Module):
+    def __init__(self, encoder_layer, num_layers, norm=None):
+        super().__init__()
+        self.layers = _get_clones(encoder_layer, num_layers)
+        self.num_layers = num_layers
+        self.norm = norm
+
+    def forward(self, src,
+                mask: Optional[Tensor] = None,
+                src_key_padding_mask: Optional[Tensor] = None,
+                pos: Optional[Tensor] = None):
+        output = src
+
+        for layer in self.layers:
+            output = layer(output, src_mask=mask,
+                           src_key_padding_mask=src_key_padding_mask, pos=pos)
+
+        if self.norm is not None:
+            output = self.norm(output)
+
+        return output
+
+
+class TransformerDecoder(nn.Module):
+    # Intermediate results are stacked at dimension 0 for aux loss
+    def __init__(self, decoder_layer, num_layers, norm=None, return_intermediate=False):
+        super().__init__()
+        self.layers = _get_clones(decoder_layer, num_layers)
+        self.num_layers = num_layers
+        self.norm = norm
+        self.return_intermediate = return_intermediate
+
+    def forward(self, tgt, memory,
+                tgt_mask: Optional[Tensor] = None,
+                memory_mask: Optional[Tensor] = None,
+                tgt_key_padding_mask: Optional[Tensor] = None,
+                memory_key_padding_mask: Optional[Tensor] = None,
+                pos: Optional[Tensor] = None,
+                query_pos: Optional[Tensor] = None):
+        output = tgt
+
+        intermediate = []
+
+        for layer in self.layers:
+            output = layer(output, memory, tgt_mask=tgt_mask,
+                           memory_mask=memory_mask,
+                           tgt_key_padding_mask=tgt_key_padding_mask,
+                           memory_key_padding_mask=memory_key_padding_mask,
+                           pos=pos, query_pos=query_pos)
+            if self.return_intermediate:
+                intermediate.append(self.norm(output))
+
+        if self.norm is not None:
+            output = self.norm(output)
+            if self.return_intermediate:
+                intermediate.pop()
+                intermediate.append(output)
+
+        if self.return_intermediate:
+            return torch.stack(intermediate)
+
+        return output.unsqueeze(0)
+
+
+class TransformerEncoderLayer(nn.Module):
+    def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1,
+                 activation="relu", normalize_before=False):
+        super().__init__()
+        self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout)
+        # Implementation of Feedforward model
+        self.linear1 = nn.Linear(d_model, dim_feedforward)
+        self.dropout = nn.Dropout(dropout)
+        self.linear2 = nn.Linear(dim_feedforward, d_model)
+
+        self.norm1 = nn.LayerNorm(d_model)
+        self.norm2 = nn.LayerNorm(d_model)
+        self.dropout1 = nn.Dropout(dropout)
+        self.dropout2 = nn.Dropout(dropout)
+
+        self.activation = _get_activation_fn(activation)
+        self.normalize_before = normalize_before
+
+    def forward_post(self,
+                     src,
+                     src_mask: Optional[Tensor] = None,
+                     src_key_padding_mask: Optional[Tensor] = None,
+                     pos: Optional[Tensor] = None):
+        q = k = _with_pos_embed(src, pos)
+        src2 = self.self_attn(q, k, value=src, attn_mask=src_mask,
+                              key_padding_mask=src_key_padding_mask)[0]
+        src = src + self.dropout1(src2)
+        src = self.norm1(src)
+        src2 = self.linear2(self.dropout(self.activation(self.linear1(src))))
+        src = src + self.dropout2(src2)
+        src = self.norm2(src)
+
+        return src
+
+    def forward_pre(self, src,
+                    src_mask: Optional[Tensor] = None,
+                    src_key_padding_mask: Optional[Tensor] = None,
+                    pos: Optional[Tensor] = None):
+        src2 = self.norm1(src)
+        q = k = _with_pos_embed(src2, pos)
+        src2 = self.self_attn(q, k, value=src2, attn_mask=src_mask,
+                              key_padding_mask=src_key_padding_mask)[0]
+        src = src + self.dropout1(src2)
+        src2 = self.norm2(src)
+        src2 = self.linear2(self.dropout(self.activation(self.linear1(src2))))
+        src = src + self.dropout2(src2)
+
+        return src
+
+    def forward(self, src,
+                src_mask: Optional[Tensor] = None,
+                src_key_padding_mask: Optional[Tensor] = None,
+                pos: Optional[Tensor] = None):
+        if self.normalize_before:
+            return self.forward_pre(src, src_mask, src_key_padding_mask, pos)
+        else:
+            return self.forward_post(src, src_mask, src_key_padding_mask, pos)
+
+
+class TransformerDecoderLayer(nn.Module):
+    def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1,
+                 activation="relu", normalize_before=False):
+        super().__init__()
+        self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout)
+        self.multihead_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout)
+        # Implementation of feed-forward model
+        self.linear1 = nn.Linear(d_model, dim_feedforward)
+        self.dropout = nn.Dropout(dropout)
+        self.linear2 = nn.Linear(dim_feedforward, d_model)
+
+        self.norm1 = nn.LayerNorm(d_model)
+        self.norm2 = nn.LayerNorm(d_model)
+        self.norm3 = nn.LayerNorm(d_model)
+        self.dropout1 = nn.Dropout(dropout)
+        self.dropout2 = nn.Dropout(dropout)
+        self.dropout3 = nn.Dropout(dropout)
+
+        self.activation = _get_activation_fn(activation)
+        self.normalize_before = normalize_before
+
+    def forward_post(self, tgt, memory,
+                     tgt_mask: Optional[Tensor] = None,
+                     memory_mask: Optional[Tensor] = None,
+                     tgt_key_padding_mask: Optional[Tensor] = None,
+                     memory_key_padding_mask: Optional[Tensor] = None,
+                     pos: Optional[Tensor] = None,
+                     query_pos: Optional[Tensor] = None):
+        q = k = _with_pos_embed(tgt, query_pos)
+        tgt2 = self.self_attn(q, k, value=tgt, attn_mask=tgt_mask,
+                              key_padding_mask=tgt_key_padding_mask)[0]
+        tgt = tgt + self.dropout1(tgt2)
+        tgt = self.norm1(tgt)
+        tgt2 = self.multihead_attn(query=_with_pos_embed(tgt, query_pos),
+                                   key=_with_pos_embed(memory, pos),
+                                   value=memory, attn_mask=memory_mask,
+                                   key_padding_mask=memory_key_padding_mask)[0]
+        tgt = tgt + self.dropout2(tgt2)
+        tgt = self.norm2(tgt)
+        tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt))))
+        tgt = tgt + self.dropout3(tgt2)
+        tgt = self.norm3(tgt)
+
+        return tgt
+
+    def forward_pre(self, tgt, memory,
+                    tgt_mask: Optional[Tensor] = None,
+                    memory_mask: Optional[Tensor] = None,
+                    tgt_key_padding_mask: Optional[Tensor] = None,
+                    memory_key_padding_mask: Optional[Tensor] = None,
+                    pos: Optional[Tensor] = None,
+                    query_pos: Optional[Tensor] = None):
+        tgt2 = self.norm1(tgt)
+        q = k = _with_pos_embed(tgt2, query_pos)
+        tgt2 = self.self_attn(q, k, value=tgt2, attn_mask=tgt_mask,
+                              key_padding_mask=tgt_key_padding_mask)[0]
+        tgt = tgt + self.dropout1(tgt2)
+        tgt2 = self.norm2(tgt)
+        tgt2 = self.multihead_attn(query=_with_pos_embed(tgt2, query_pos),
+                                   key=_with_pos_embed(memory, pos),
+                                   value=memory, attn_mask=memory_mask,
+                                   key_padding_mask=memory_key_padding_mask)[0]
+        tgt = tgt + self.dropout2(tgt2)
+        tgt2 = self.norm3(tgt)
+        tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt2))))
+        tgt = tgt + self.dropout3(tgt2)
+
+        return tgt
+
+    def forward(self, tgt, memory,
+                tgt_mask: Optional[Tensor] = None,
+                memory_mask: Optional[Tensor] = None,
+                tgt_key_padding_mask: Optional[Tensor] = None,
+                memory_key_padding_mask: Optional[Tensor] = None,
+                pos: Optional[Tensor] = None,
+                query_pos: Optional[Tensor] = None):
+        if self.normalize_before:
+            return self.forward_pre(tgt, memory, tgt_mask, memory_mask,
+                                    tgt_key_padding_mask, memory_key_padding_mask, pos, query_pos)
+        else:
+            return self.forward_post(tgt, memory, tgt_mask, memory_mask,
+                                     tgt_key_padding_mask, memory_key_padding_mask, pos, query_pos)
+
+
+def _get_clones(module, N):
+    return nn.ModuleList([copy.deepcopy(module) for _ in range(N)])
+
+
+def build_transformer(hidden_dim,
+                      dropout,
+                      nheads,
+                      dim_feedforward,
+                      enc_layers,
+                      dec_layers,
+                      pre_norm=False,
+                      return_intermediate_dec=False):
+
+    return Transformer(
+        d_model=hidden_dim,
+        dropout=dropout,
+        nhead=nheads,
+        dim_feedforward=dim_feedforward,
+        num_encoder_layers=enc_layers,
+        num_decoder_layers=dec_layers,
+        normalize_before=pre_norm,
+        return_intermediate_dec=return_intermediate_dec,
+    )
+
+
+def _with_pos_embed(tensor, pos: Optional[Tensor]):
+    return tensor if pos is None else tensor + pos
+
+
+def _get_activation_fn(activation):
+    """Return an activation function given a string"""
+    if activation == "relu":
+        return F.relu
+    elif activation == "gelu":
+        return F.gelu
+    elif activation == "glu":
+        return F.glu
+    else:
+        raise RuntimeError(F"activation should be relu/gelu, not {activation}.")

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/utils.py

@@ -0,0 +1,4 @@
+try:
+    from torch.hub import load_state_dict_from_url
+except ImportError:
+    from torch.utils.model_zoo import load_url as load_state_dict_from_url

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/vgg_encoder.py

@@ -0,0 +1,32 @@
+from torch import nn as nn
+from torchvision.models import vgg16_bn
+
+from .builder import MODELS
+
+
+@MODELS.register()
+class VGG16(nn.Module):
+    # Modified VGG16 backbone in DeepLab-LargeFOV,
+    # note that due to legacy implementation issues,
+    # the converted fully-connected layers (FC-6 FC-7) are not included here.
+    # jcdubron/scnn_pytorch
+
+    def __init__(self, pretrained=True):
+        super().__init__()
+        self.pretrained = pretrained
+        self.net = vgg16_bn(pretrained=self.pretrained).features
+        for i in [34, 37, 40]:
+            conv = self.net._modules[str(i)]
+            dilated_conv = nn.Conv2d(
+                conv.in_channels, conv.out_channels, conv.kernel_size, stride=conv.stride,
+                padding=tuple(p * 2 for p in conv.padding), dilation=2, bias=(conv.bias is not None)
+            )
+            dilated_conv.load_state_dict(conv.state_dict())
+            self.net._modules[str(i)] = dilated_conv
+        self.net._modules.pop('33')
+        self.net._modules.pop('43')
+
+    def forward(self, x):
+        x = self.net(x)
+
+        return x

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/onnx_utils.py

@@ -0,0 +1,99 @@
+# Convert only the pt model part
+
+import onnx
+import onnxruntime as ort
+import numpy as np
+import torch
+
+
+DEFAULT_OPSET_VERSION = 9
+MINIMAL_OPSET_VERSIONS = {
+    # Others use 9
+    'LSTR': 10,
+    'RESA': 11,
+    'SpatialConv': 11,
+    'SwinTransformer': 11,
+    'LaneAtt': 11
+}
+TRACE_REQUIRE_PREPROCESSING = [
+    'LSTR',
+    'RESA',
+    'RESA_Net',
+    'LaneAtt'
+]
+
+
+def get_minimal_opset_version(cfg, min_version):
+    # Recursively get minimum version
+    if isinstance(cfg, dict):
+        for k, v in cfg.items():
+            if k == 'name':
+                temp = MINIMAL_OPSET_VERSIONS.get(v)
+                if temp is None:
+                    temp = DEFAULT_OPSET_VERSION
+                min_version = max(min_version, temp)
+            else:
+                min_version = max(min_version, get_minimal_opset_version(v, min_version))
+        return min_version
+    else:
+        return DEFAULT_OPSET_VERSION
+
+
+def append_trace_arg(cfg, trace_arg):
+    # Do the above trick again
+    if isinstance(cfg, dict):
+        if cfg.get('name') in TRACE_REQUIRE_PREPROCESSING:
+            cfg['trace_arg'] = trace_arg
+        else:
+            for k in cfg.keys():
+                cfg[k] = append_trace_arg(cfg[k], trace_arg)
+
+    return cfg
+
+
+def pt_to_onnx(net, dummy, filename, opset_version=9):
+    try:
+        net.eval(profiling=True)
+    except TypeError:
+        net.eval()
+    temp = net(dummy)
+    torch.onnx.export(net, dummy, filename, verbose=True, input_names=['input1'], output_names=temp.keys(),
+                      opset_version=opset_version)
+
+
+@torch.no_grad()
+def test_conversion(pt_net, onnx_filename, dummy):
+    try:
+        pt_net.eval(profiling=True)
+    except TypeError:
+        pt_net.eval()
+    pt_out = pt_net(dummy)
+    dummy = dummy.cpu()
+    onnx_out = inference_onnx(dummy, onnx_filename)
+    diff = 0.0
+    avg = 0.0
+    for k, temp_pt in pt_out.items():
+        temp_onnx = onnx_out[k]
+        diff += np.abs((temp_onnx - temp_pt.cpu().numpy())).mean()
+        avg += temp_pt.abs().mean().item()
+    diff /= len(onnx_out)
+    avg /= len(onnx_out)
+    diff_percentage = diff / avg * 100
+    print('Average diff: {}\nAverage diff (%): {}'.format(diff, diff_percentage))
+    assert diff_percentage < 0.1, 'Diff over 0.1%, please check for special operators!'
+
+
+def inference_onnx(dummy, onnx_filename):
+    onnx_net = onnx.load(onnx_filename)
+    onnx.checker.check_model(onnx_net)
+    onnx.helper.printable_graph(onnx_net.graph)
+    providers = ["CUDAExecutionProvider", "CPUExecutionProvider"]
+    ort_session = ort.InferenceSession(onnx_filename, providers=providers)
+    print(ort_session.get_providers())
+    input_all = [node.name for node in onnx_net.graph.input]
+    input_initializer = [node.name for node in onnx_net.graph.initializer]
+    input_names = list(set(input_all) - set(input_initializer))
+    onnx_out = ort_session.run(None, {input_names[0]: dummy.numpy()})
+    output_names = [node.name for node in onnx_net.graph.output]
+
+    return {k: v for k, v in zip(output_names, onnx_out)}