feat: initial HSAP platform

Huaxu Sentinel Active Safety Platform with embedded algorithm code,
Docker Compose setup, and vendored dataset scaffolds for clone-and-run.

Co-authored-by: Cursor <cursoragent@cursor.com>

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/__init__.py

@@ -0,0 +1,17 @@
+# Some codes may look like copy-paste, but every line of code is manually checked
+# Implementation reference order (not strict):
+# Paper Content = Official Impl > TorchVision Impl > OpenMMLab Impl > Community Impl
+# If there exists an official improved code version that is different from paper content,
+# probably it will not be used here in PytorchAutoDrive.
+from .builder import MODELS
+from . import segmentation
+from . import lane_detection
+from . import transformer
+from . import common_models
+from .backbone_wrappers import free_resnet_backbone, predefined_resnet_backbone
+from .vgg_encoder import VGG16
+from .mobilenet_v2 import MobileNetV2Encoder
+from .mobilenet_v3 import MobileNetV3Encoder
+from .rep_vgg import RepVggEncoder
+from .swin import SwinTransformer
+from .erfnet_encoder import ERFNetEncoder  # add ERFNetEncoder

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/_utils.py

@@ -0,0 +1,100 @@
+from collections import OrderedDict
+
+import torch
+from torch import nn
+
+
+def is_tracing() -> bool:
+    # https://github.com/pytorch/pytorch/issues/42448
+    if torch.__version__ >= '1.7.0':
+        return torch.jit.is_tracing()
+    elif torch.__version__ >= '1.6.0':
+        return torch._C._is_tracing()
+    else:
+        return False
+
+
+class IntermediateLayerGetter(nn.ModuleDict):
+    """
+    Module wrapper that returns intermediate layers from a model
+
+    It has a strong assumption that the modules have been registered
+    into the model in the same order as they are used.
+    This means that one should **not** reuse the same nn.Module
+    twice in the forward if you want this to work.
+
+    Additionally, it is only able to query submodules that are directly
+    assigned to the model. So if `model` is passed, `model.feature1` can
+    be returned, but not `model.feature1.layer2`.
+
+    Arguments:
+        model (nn.Module): model on which we will extract the features
+        return_layers (Dict[name, new_name]): a dict containing the names
+            of the modules for which the activations will be returned as
+            the key of the dict, and the value of the dict is the name
+            of the returned activation (which the user can specify).
+
+    Examples::
+
+        >>> m = torchvision.models.resnet18(pretrained=True)
+        >>> # extract layer1 and layer3, giving as names `feat1` and feat2`
+        >>> new_m = torchvision.models._utils.IntermediateLayerGetter(m,
+        >>>     {'layer1': 'feat1', 'layer3': 'feat2'})
+        >>> out = new_m(torch.rand(1, 3, 224, 224))
+        >>> print([(k, v.shape) for k, v in out.items()])
+        >>>     [('feat1', torch.Size([1, 64, 56, 56])),
+        >>>      ('feat2', torch.Size([1, 256, 14, 14]))]
+    """
+    def __init__(self, model, return_layers):
+        if not set(return_layers).issubset([name for name, _ in model.named_children()]):
+            raise ValueError("return_layers are not present in model")
+
+        orig_return_layers = return_layers
+        return_layers = {k: v for k, v in return_layers.items()}
+        layers = OrderedDict()
+        for name, module in model.named_children():
+            layers[name] = module
+            if name in return_layers:
+                del return_layers[name]
+            if not return_layers:
+                break
+
+        super(IntermediateLayerGetter, self).__init__(layers)
+        self.return_layers = orig_return_layers
+
+    def forward(self, x):
+        out = OrderedDict()
+        for name, module in self.named_children():
+            x = module(x)
+            if name in self.return_layers:
+                out_name = self.return_layers[name]
+                out[out_name] = x
+        return out
+
+
+# Copied from OpenMMLab.
+# https://github.com/open-mmlab/mmsegmentation/blob/fa8c93e78c1679eee558ced5047cc280adfa1c1d/mmseg/models/utils/make_divisible.py#L2
+def make_divisible(value, divisor, min_value=None, min_ratio=0.9):
+    """Make divisible function.
+    This function rounds the channel number to the nearest value that can be
+    divisible by the divisor. It is taken from the original tf repo. It ensures
+    that all layers have a channel number that is divisible by divisor. It can
+    be seen here: https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py  # noqa
+    Args:
+        value (int): The original channel number.
+        divisor (int): The divisor to fully divide the channel number.
+        min_value (int): The minimum value of the output channel.
+            Default: None, means that the minimum value equal to the divisor.
+        min_ratio (float): The minimum ratio of the rounded channel number to
+            the original channel number. Default: 0.9.
+    Returns:
+        int: The modified output channel number.
+    """
+
+    if min_value is None:
+        min_value = divisor
+    new_value = max(min_value, int(value + divisor / 2) // divisor * divisor)
+    # Make sure that round down does not go down by more than (1-min_ratio).
+    if new_value < min_ratio * value:
+        new_value += divisor
+    return new_value

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/backbone_wrappers.py

@@ -0,0 +1,39 @@
+from . import resnet
+from .resnet import _resnet, Bottleneck, BasicBlock
+from .builder import MODELS
+from ._utils import IntermediateLayerGetter
+
+
+# Maybe it is safe to assume no new block types
+block_map = {
+    'BasicBlock': BasicBlock,
+    'BottleNeck': Bottleneck
+}
+
+
+def parse_return_layers(return_layer):
+    if isinstance(return_layer, str):
+        return {return_layer: 'out'}
+    elif isinstance(return_layer, dict):
+        return return_layer
+    else:
+        raise TypeError('return_layer can either be direct dict or a string, not {}'.format(type(return_layer)))
+
+
+@MODELS.register()
+def free_resnet_backbone(arch, block, layers, pretrained, return_layer, **kwargs):
+    block = block_map[block]
+    net = _resnet(arch, block, layers, pretrained, progress=True, **kwargs)
+    return_layers = parse_return_layers(return_layer)
+
+    return IntermediateLayerGetter(net, return_layers=return_layers)
+
+
+@MODELS.register()
+def predefined_resnet_backbone(backbone_name, return_layer, **kwargs):
+    backbone = resnet.__dict__[backbone_name](**kwargs)
+    return_layers = parse_return_layers(return_layer)
+
+    return IntermediateLayerGetter(backbone, return_layers=return_layers)
+
+

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/builder.py

@@ -0,0 +1,3 @@
+from ..registry import SimpleRegistry
+
+MODELS = SimpleRegistry()

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/__init__.py

@@ -0,0 +1,3 @@
+from .blocks import *
+from .heads import *
+from .plugins import *

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/blocks/__init__.py

@@ -0,0 +1,3 @@
+from .inverted_residual import InvertedResidual, InvertedResidualV3
+from .non_bottleneck_1d import non_bottleneck_1d
+from .dilated_bottleneck import DilatedBottleneck

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/blocks/dilated_bottleneck.py

@@ -0,0 +1,42 @@
+import torch
+import torch.nn as nn
+
+from ...builder import MODELS
+
+
+@MODELS.register()
+class DilatedBottleneck(nn.Module):
+    # Refactored from https://github.com/chensnathan/YOLOF/blob/master/yolof/modeling/encoder.py
+    # Diff from typical ResNetV1.5 BottleNeck:
+    # flexible expansion rate, forbids downsampling, relu immediately after last conv
+    def __init__(self,
+                 in_channels=512,
+                 mid_channels=128,
+                 dilation=1):
+        super().__init__()
+        self.conv1 = nn.Sequential(
+            nn.Conv2d(in_channels, mid_channels, kernel_size=1, padding=0),
+            nn.BatchNorm2d(mid_channels)
+        )
+        self.conv2 = nn.Sequential(
+            nn.Conv2d(mid_channels, mid_channels,
+                      kernel_size=3, padding=dilation, dilation=dilation),
+            nn.BatchNorm2d(mid_channels)
+        )
+        self.conv3 = nn.Sequential(
+            nn.Conv2d(mid_channels, in_channels, kernel_size=1, padding=0),
+            nn.BatchNorm2d(in_channels)
+        )
+        self.relu = nn.ReLU(inplace=True)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        identity = x
+        out = self.conv1(x)
+        out = self.relu(out)
+        out = self.conv2(out)
+        out = self.relu(out)
+        out = self.conv3(out)
+        out = self.relu(out)
+        out = out + identity
+
+        return out

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/blocks/inverted_residual.py

@@ -0,0 +1,130 @@
+from torch import nn as nn
+
+from ..plugins import SELayer
+
+
+class InvertedResidual(nn.Module):
+    """InvertedResidual block for MobileNetV2.
+    Args:
+        in_channels (int): The input channels of the InvertedResidual block.
+        out_channels (int): The output channels of the InvertedResidual block.
+        stride (int): Stride of the middle (first) 3x3 convolution.
+        expand_ratio (int): Adjusts number of channels of the hidden layer
+            in InvertedResidual by this amount.
+        dilation (int): Dilation rate of depthwise conv. Default: 1
+    Returns:
+        Tensor: The output tensor.
+    """
+
+    def __init__(self, in_channels, out_channels, stride, expand_ratio, dilation=1, bias=False):
+        super(InvertedResidual, self).__init__()
+        self.stride = stride
+        assert stride in [1, 2], f'stride must in [1, 2]. ' \
+                                 f'But received {stride}.'
+        self.use_res_connect = self.stride == 1 and in_channels == out_channels
+        hidden_dim = int(round(in_channels * expand_ratio))
+        layers = []
+        if expand_ratio != 1:
+            layers.extend([
+                nn.Conv2d(in_channels=in_channels, out_channels=hidden_dim, kernel_size=1, bias=bias),
+                nn.BatchNorm2d(hidden_dim),
+                nn.ReLU6()  # min(max(0, x), 6)
+            ])
+
+        layers.extend([
+            nn.Conv2d(in_channels=hidden_dim, out_channels=hidden_dim, kernel_size=3, stride=stride,
+                      padding=dilation, dilation=dilation, groups=hidden_dim, bias=bias),
+            nn.BatchNorm2d(hidden_dim),
+            nn.ReLU6(),
+            nn.Conv2d(in_channels=hidden_dim, out_channels=out_channels, kernel_size=1, bias=bias),
+            nn.BatchNorm2d(out_channels)
+        ])
+        self.conv = nn.Sequential(*layers)
+
+    def forward(self, x):
+
+        def _inner_forward(x):
+            if self.use_res_connect:
+                return x + self.conv(x)
+            else:
+                return self.conv(x)
+
+        out = _inner_forward(x)
+
+        return out
+
+
+class InvertedResidualV3(nn.Module):
+    """Inverted Residual Block for MobileNetV3.
+    Args:
+        in_channels (int): The input channels of this Module.
+        out_channels (int): The output channels of this Module.
+        mid_channels (int): The input channels of the depthwise convolution.
+        kernel_size (int): The kernel size of the depthwise convolution. Default: 3.
+        stride (int): The stride of the depthwise convolution. Default: 1.
+        with_se (dict): with or without se layer. Default: False, which means no se layer.
+        with_expand_conv (bool): Use expand conv or not. If set False,
+            mid_channels must be the same with in_channels. Default: True.
+        act_cfg (dict): Config dict for activation layer.
+            Default: dict(type='ReLU').
+    Returns:
+        Tensor: The output tensor.
+    """
+
+    def __init__(self, in_channels, out_channels, mid_channels, kernel_size=3, stride=1, with_se=False,
+                 with_expand_conv=True, act='HSwish', bias=False, dilation=1):
+        super(InvertedResidualV3, self).__init__()
+        self.with_res_shortcut = (stride == 1 and in_channels == out_channels)
+        assert stride in [1, 2]
+        activation_layer = nn.Hardswish if act == 'HSwish' else nn.ReLU6
+        self.with_se = with_se
+        self.with_expand_conv = with_expand_conv
+        if not self.with_expand_conv:
+            assert mid_channels == in_channels
+        if self.with_expand_conv:
+            self.expand_conv = nn.Sequential(
+                nn.Conv2d(in_channels=in_channels, out_channels=mid_channels, kernel_size=1, stride=1, padding=0,
+                          bias=bias),
+                nn.BatchNorm2d(mid_channels),
+                activation_layer()
+            )
+        if stride > 1 and dilation > 1:
+            raise ValueError('Can\'t have stride and dilation both > 1 in MobileNetV3')
+        self.depthwise_conv = nn.Sequential(
+            nn.Conv2d(in_channels=mid_channels, out_channels=mid_channels, kernel_size=kernel_size, stride=stride,
+                      padding=(kernel_size - 1) // 2 * dilation, dilation=dilation, groups=mid_channels, bias=bias),
+            nn.BatchNorm2d(mid_channels),
+            activation_layer()
+        )
+        if self.with_se:
+            self.se = SELayer(channels=mid_channels, ratio=4)
+
+        self.linear_conv = nn.Sequential(
+            nn.Conv2d(in_channels=mid_channels, out_channels=out_channels, kernel_size=1, stride=1, padding=0,
+                      bias=False),
+            nn.BatchNorm2d(out_channels)
+        )
+
+    def forward(self, x):
+
+        def _inner_forward(x):
+            out = x
+
+            if self.with_expand_conv:
+                out = self.expand_conv(out)
+
+            out = self.depthwise_conv(out)
+
+            if self.with_se:
+                out = self.se(out)
+
+            out = self.linear_conv(out)
+
+            if self.with_res_shortcut:
+                return x + out
+            else:
+                return out
+
+        out = _inner_forward(x)
+
+        return out

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/blocks/non_bottleneck_1d.py

@@ -0,0 +1,34 @@
+# ERFNet's residual block
+from torch import nn as nn
+from torch.nn import functional as F
+
+
+class non_bottleneck_1d(nn.Module):
+    def __init__(self, chann, dropprob, dilated):
+        super().__init__()
+        self.conv3x1_1 = nn.Conv2d(chann, chann, (3, 1), stride=1, padding=(1, 0), bias=True)
+        self.conv1x3_1 = nn.Conv2d(chann, chann, (1, 3), stride=1, padding=(0, 1), bias=True)
+        self.bn1 = nn.BatchNorm2d(chann, eps=1e-03)
+        self.conv3x1_2 = nn.Conv2d(chann, chann, (3, 1), stride=1, padding=(1 * dilated, 0),
+                                   bias=True, dilation=(dilated, 1))
+        self.conv1x3_2 = nn.Conv2d(chann, chann, (1, 3), stride=1, padding=(0, 1 * dilated),
+                                   bias=True, dilation=(1, dilated))
+        self.bn2 = nn.BatchNorm2d(chann, eps=1e-03)
+        self.dropout = nn.Dropout2d(dropprob)
+
+    def forward(self, input):
+        output = self.conv3x1_1(input)
+        output = F.relu(output)
+        output = self.conv1x3_1(output)
+        output = self.bn1(output)
+        output = F.relu(output)
+
+        output = self.conv3x1_2(output)
+        output = F.relu(output)
+        output = self.conv1x3_2(output)
+        output = self.bn2(output)
+
+        if self.dropout.p != 0:
+            output = self.dropout(output)
+
+        return F.relu(output + input)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/heads/__init__.py

@@ -0,0 +1,7 @@
+from .busd import BUSD
+from .scnn_decoder import SCNNDecoder
+from .plain_decoder import PlainDecoder
+from .simple_lane_exist import SimpleLaneExist
+from .encoder_decoder_lane_exist import EDLaneExist
+from .resa_lane_exist import RESALaneExist
+from .uper_head import UperHead

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/heads/busd.py

@@ -0,0 +1,57 @@
+from torch import nn as nn
+from torch.nn import functional as F
+
+from ...builder import MODELS
+from ..blocks import non_bottleneck_1d
+
+
+class BilateralUpsamplerBlock(nn.Module):
+    # Added a coarse path to the original ERFNet UpsamplerBlock
+    # Copied and modified from:
+    # https://github.com/ZJULearning/resa/blob/14b0fea6a1ab4f45d8f9f22fb110c1b3e53cf12e/models/decoder.py#L67
+
+    def __init__(self, ninput, noutput):
+        super(BilateralUpsamplerBlock, self).__init__()
+        self.conv = nn.ConvTranspose2d(ninput, noutput, 3, stride=2, padding=1, output_padding=1, bias=True)
+        self.bn = nn.BatchNorm2d(noutput, eps=1e-3, track_running_stats=True)
+        self.follows = nn.ModuleList(non_bottleneck_1d(noutput, 0, 1) for _ in range(2))
+
+        # interpolate
+        self.interpolate_conv = nn.Conv2d(ninput, noutput, kernel_size=1, bias=False)
+        self.interpolate_bn = nn.BatchNorm2d(noutput, eps=1e-3)
+
+    def forward(self, input):
+        # Fine branch
+        output = self.conv(input)
+        output = self.bn(output)
+        out = F.relu(output)
+        for follow in self.follows:
+            out = follow(out)
+
+        # Coarse branch (keep at align_corners=True)
+        interpolate_output = self.interpolate_conv(input)
+        interpolate_output = self.interpolate_bn(interpolate_output)
+        interpolate_output = F.relu(interpolate_output)
+        interpolated = F.interpolate(interpolate_output, size=out.shape[-2:], mode='bilinear', align_corners=True)
+
+        return out + interpolated
+
+
+@MODELS.register()
+class BUSD(nn.Module):
+    # Bilateral Up-Sampling Decoder in RESA paper,
+    # make it work for arbitrary input channels (8x up-sample then predict).
+    # Drops transposed prediction layer in ERFNet, while adds an extra up-sampling block.
+
+    def __init__(self, in_channels=128, num_classes=5):
+        super(BUSD, self).__init__()
+        base = in_channels // 8
+        self.layers = nn.ModuleList(BilateralUpsamplerBlock(ninput=base * 2 ** (3 - i), noutput=base * 2 ** (2 - i))
+                                    for i in range(3))
+        self.output_proj = nn.Conv2d(base, num_classes, kernel_size=1, bias=True)  # Keep bias=True for prediction
+
+    def forward(self, x):
+        for layer in self.layers:
+            x = layer(x)
+
+        return self.output_proj(x)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/heads/encoder_decoder_lane_exist.py

@@ -0,0 +1,50 @@
+from torch import nn as nn
+from torch.nn import functional as F
+
+from ...builder import MODELS
+
+
+@MODELS.register()
+class EDLaneExist(nn.Module):
+    # Lane exist head for ERFNet, ENet
+    # Really tricky without global pooling
+
+    def __init__(self, num_output, flattened_size=3965, dropout=0.1, pool='avg'):
+        super().__init__()
+
+        self.layers = nn.ModuleList()
+        self.layers.append(nn.Conv2d(128, 32, (3, 3), stride=1, padding=(4, 4), bias=False, dilation=(4, 4)))
+        self.layers.append(nn.BatchNorm2d(32, eps=1e-03))
+
+        self.layers_final = nn.ModuleList()
+        self.layers_final.append(nn.Dropout2d(dropout))
+        self.layers_final.append(nn.Conv2d(32, 5, (1, 1), stride=1, padding=(0, 0), bias=True))
+
+        if pool == 'max':
+            self.pool = nn.MaxPool2d(2, stride=2)
+        elif pool == 'avg':
+            self.pool = nn.AvgPool2d(2, stride=2)
+        else:
+            raise RuntimeError("This type of pool has not been defined yet!")
+
+        self.linear1 = nn.Linear(flattened_size, 128)
+        self.linear2 = nn.Linear(128, num_output)
+
+    def forward(self, input):
+        output = input
+        for layer in self.layers:
+            output = layer(output)
+
+        output = F.relu(output)
+
+        for layer in self.layers_final:
+            output = layer(output)
+
+        output = F.softmax(output, dim=1)
+        output = self.pool(output)
+        output = output.flatten(start_dim=1)
+        output = self.linear1(output)
+        output = F.relu(output)
+        output = self.linear2(output)
+
+        return output

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/heads/plain_decoder.py

@@ -0,0 +1,19 @@
+from torch import nn as nn
+
+from ...builder import MODELS
+
+
+@MODELS.register()
+class PlainDecoder(nn.Module):
+    # Plain decoder (albeit simplest) from the RESA paper
+
+    def __init__(self, in_channels=128, num_classes=5):
+        super(PlainDecoder, self).__init__()
+        self.dropout1 = nn.Dropout2d(0.1)
+        self.conv1 = nn.Conv2d(in_channels, num_classes, 1, bias=True)
+
+    def forward(self, x):
+        x = self.dropout1(x)
+        x = self.conv1(x)
+
+        return x

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/heads/resa_lane_exist.py

@@ -0,0 +1,30 @@
+from torch import nn as nn
+from torch.nn import functional as F
+
+from ...builder import MODELS
+
+
+@MODELS.register()
+class RESALaneExist(nn.Module):
+    def __init__(self, num_output, flattened_size=3965, dropout=0.1, in_channels=128):
+        super().__init__()
+
+        self.layers = nn.ModuleList()
+        self.layers.append(nn.Dropout2d(dropout))
+        self.layers.append(nn.Conv2d(in_channels, num_output + 1, (1, 1), stride=1, padding=(0, 0), bias=True))
+        self.pool = nn.AvgPool2d(2, stride=2)
+        self.linear1 = nn.Linear(flattened_size, 128)
+        self.linear2 = nn.Linear(128, num_output)
+
+    def forward(self, input):
+        output = input
+        for layer in self.layers:
+            output = layer(output)
+        output = F.softmax(output, dim=1)
+        output = self.pool(output)
+        output = output.flatten(start_dim=1)
+        output = self.linear1(output)
+        output = F.relu(output)
+        output = self.linear2(output)
+
+        return output

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/heads/scnn_decoder.py

@@ -0,0 +1,32 @@
+from torch import nn as nn
+from torch.nn import functional as F
+
+from ...builder import MODELS
+
+
+@MODELS.register()
+class SCNNDecoder(nn.Module):
+    # Unused
+    # SCNN original decoder for ResNet-101, very large channels, maybe impossible to add anything
+    # resnet-101 -> H x W x 2048
+    # 3x3 Conv -> H x W x 512
+    # Dropout 0.1
+    # 1x1 Conv -> H x W x 5
+    # https://github.com/XingangPan/SCNN/issues/35
+
+    def __init__(self, in_channels=2048, num_classes=5):
+        super(SCNNDecoder, self).__init__()
+        out_channels = in_channels // 4
+        self.conv1 = nn.Conv2d(in_channels, out_channels, 3, padding=1, bias=False)
+        self.bn1 = nn.BatchNorm2d(out_channels)
+        self.dropout1 = nn.Dropout2d(0.1)
+        self.conv2 = nn.Conv2d(out_channels, num_classes, 1, bias=False)
+
+    def forward(self, x):
+        x = self.conv1(x)
+        x = self.bn1(x)
+        x = F.relu(x)
+        x = self.dropout1(x)
+        x = self.conv2(x)
+
+        return x

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/heads/simple_lane_exist.py

@@ -0,0 +1,28 @@
+import torch
+from torch import nn as nn
+from torch.nn import functional as F
+
+from ...builder import MODELS
+
+
+@MODELS.register()
+class SimpleLaneExist(nn.Module):
+    # Typical lane existence head originated from the SCNN paper
+
+    def __init__(self, num_output, flattened_size=4500):
+        super().__init__()
+        self.avgpool = nn.AvgPool2d(2, 2)
+        self.linear1 = nn.Linear(flattened_size, 128)
+        self.linear2 = nn.Linear(128, num_output)
+
+    def forward(self, input, predict=False):
+        # input: logits
+        output = self.avgpool(input)
+        output = output.flatten(start_dim=1)
+        output = self.linear1(output)
+        output = F.relu(output)
+        output = self.linear2(output)
+        if predict:
+            output = torch.sigmoid(output)
+
+        return output

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/heads/uper_head.py

@@ -0,0 +1,77 @@
+import torch
+import torch.nn as nn
+from torch.nn import functional as F
+
+from ...builder import MODELS
+from ..plugins.ppm import PPM
+
+
+@MODELS.register()
+class UperHead(nn.Module):
+    def __init__(self, in_channels, channels, pool_scales=(1, 2, 3, 6), align_corners=False):
+        super(UperHead, self).__init__()
+        self.in_channels = in_channels
+        self.channels = channels
+        self.align_corners = align_corners
+        # PSP module
+        self.psp_modules = PPM(pool_scales=pool_scales, in_channels=self.in_channels[-1], channels=self.channels,
+                               align_corners=align_corners)
+        self.psp_bottleneck = nn.Sequential(
+            nn.Conv2d(in_channels=self.in_channels[-1] + len(pool_scales) * self.channels, out_channels=self.channels,
+                      kernel_size=3, padding=1),
+            nn.BatchNorm2d(self.channels),
+            nn.ReLU()
+        )
+        # FPN module
+        self.lateral_convs = nn.ModuleList()
+        self.fpn_convs = nn.ModuleList()
+        for in_channel in self.in_channels[:-1]:
+            lateral_conv = nn.Sequential(
+                nn.Conv2d(in_channel, self.channels, 1),
+                nn.BatchNorm2d(self.channels),
+                nn.ReLU())
+            fpn_conv = nn.Sequential(
+                nn.Conv2d(self.channels, self.channels, 3, padding=1),
+                nn.BatchNorm2d(self.channels),
+                nn.ReLU()
+            )
+            self.lateral_convs.append(lateral_conv)
+            self.fpn_convs.append(fpn_conv)
+        self.fpn_bottleneck = nn.Sequential(
+            nn.Conv2d(len(self.in_channels) * self.channels, self.channels, 3, padding=1),
+            nn.BatchNorm2d(self.channels),
+            nn.ReLU()
+        )
+
+    def psp_forward(self, inputs):
+        # forward function for psp module
+        x = inputs[-1]
+        psp_outs = [x]
+        psp_outs.extend(self.psp_modules(x))
+        psp_outs = torch.cat(psp_outs, dim=1)
+        outputs = self.psp_bottleneck(psp_outs)
+
+        return outputs
+
+    def forward(self, inputs):
+        assert isinstance(inputs, tuple), 'inputs must be a tuple'
+        inputs = list(inputs)
+        # build laterals
+        laterals = [lateral_conv(inputs[i]) for i, lateral_conv in enumerate(self.lateral_convs)]
+        laterals.append(self.psp_forward(inputs))
+        # build top-down path
+        used_backbone_levels = len(laterals)
+        for i in range(used_backbone_levels - 1, 0, -1):
+            prev_shape = laterals[i - 1].shape[2:]
+            laterals[i - 1] = laterals[i - 1] + F.interpolate(laterals[i], size=prev_shape, mode='bilinear',
+                                                              align_corners=self.align_corners)
+        # build outputs
+        fpn_outs = [self.fpn_convs[i](laterals[i]) for i in range(used_backbone_levels - 1)]
+        # add psp feature
+        fpn_outs.append(laterals[-1])
+        for i in range(used_backbone_levels - 1, 0, -1):
+            fpn_outs[i] = F.interpolate(fpn_outs[i], fpn_outs[0].shape[2:], mode='bilinear',
+                                       align_corners=self.align_corners)
+        fpn_outs = torch.cat(fpn_outs, dim=1)
+        output = self.fpn_bottleneck(fpn_outs)
+        return output

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/plugins/__init__.py

@@ -0,0 +1,7 @@
+from .resa_layer import RESA
+from .resa_reducer import RESAReducer
+from .se import SELayer
+from .spatial_conv import SpatialConv
+from .conv_projection_1d import ConvProjection_1D
+from .feature_flip_fusion import FeatureFlipFusion, DCN_v2_Ref
+from .dilated_blocks import predefined_dilated_blocks

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/plugins/conv_projection_1d.py

@@ -0,0 +1,24 @@
+import torch
+import torch.nn as nn
+from torch.nn import functional as F
+
+from ...builder import MODELS
+
+
+@MODELS.register()
+class ConvProjection_1D(torch.nn.Module):
+    # Projection based on line features (1D convs)
+    def __init__(self, num_layers, in_channels, bias=True, k=3):
+        # bias is set as True in FCOS
+        super().__init__()
+        self.num_layers = num_layers
+        self.hidden_layers = nn.ModuleList(nn.Conv1d(in_channels if i > 0 else in_channels,
+                                                     in_channels, kernel_size=k, bias=bias, padding=(k - 1) // 2)
+                                           for i in range(num_layers))
+        self.hidden_norms = nn.ModuleList(nn.BatchNorm1d(in_channels) for _ in range(num_layers))
+
+    def forward(self, x):
+        for conv, norm in zip(self.hidden_layers, self.hidden_norms):
+            x = F.relu(norm(conv(x)))
+
+        return x

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/plugins/dilated_blocks.py

@@ -0,0 +1,18 @@
+import torch.nn as nn
+
+from ...builder import MODELS
+
+
+@MODELS.register()
+def predefined_dilated_blocks(in_channels, mid_channels, dilations):
+    # As in YOLOF
+    blocks = [MODELS.from_dict(
+        dict(
+            name='DilatedBottleneck',
+            in_channels=in_channels,
+            mid_channels=mid_channels,
+            dilation=d
+        )
+    ) for d in dilations]
+
+    return nn.Sequential(*blocks)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/plugins/feature_flip_fusion.py

@@ -0,0 +1,78 @@
+import torch
+import torch.nn as nn
+from mmcv.ops import ModulatedDeformConv2d, modulated_deform_conv2d  # DCNv2
+from torch.nn import functional as F
+from ...builder import MODELS
+
+
+@MODELS.register()
+class DCN_v2_Ref(ModulatedDeformConv2d):
+    """A Encapsulation that acts as normal Conv
+    layers. Modified from mmcv's DCNv2.
+
+    Args:
+        in_channels (int): Same as nn.Conv2d.
+        out_channels (int): Same as nn.Conv2d.
+        kernel_size (int or tuple[int]): Same as nn.Conv2d.
+        stride (int): Same as nn.Conv2d, while tuple is not supported.
+        padding (int): Same as nn.Conv2d, while tuple is not supported.
+        dilation (int): Same as nn.Conv2d, while tuple is not supported.
+        groups (int): Same as nn.Conv2d.
+        bias (bool or str): If specified as `auto`, it will be decided by the
+            norm_cfg. Bias will be set as True if norm_cfg is None, otherwise
+            False.
+    """
+
+    _version = 2
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.conv_offset = nn.Conv2d(
+            self.in_channels * 2,
+            self.deform_groups * 3 * self.kernel_size[0] * self.kernel_size[1],
+            kernel_size=self.kernel_size,
+            stride=self.stride,
+            padding=self.padding,
+            dilation=self.dilation,
+            bias=True)
+        self.init_weights()
+
+    def init_weights(self):
+        super().init_weights()
+        if hasattr(self, 'conv_offset'):
+            self.conv_offset.weight.data.zero_()
+            self.conv_offset.bias.data.zero_()
+
+    def forward(self, x, ref):
+        concat = torch.cat([x, ref], dim=1)
+        out = self.conv_offset(concat)
+        o1, o2, mask = torch.chunk(out, 3, dim=1)
+        offset = torch.cat((o1, o2), dim=1)
+        mask = torch.sigmoid(mask)
+
+        return modulated_deform_conv2d(x, offset, mask, self.weight, self.bias,
+                                       self.stride, self.padding,
+                                       self.dilation, self.groups,
+                                       self.deform_groups)
+
+
+@MODELS.register()
+class FeatureFlipFusion(nn.Module):
+    def __init__(self, channels):
+        super().__init__()
+        self.proj1 = nn.Sequential(
+            nn.Conv2d(channels, channels, kernel_size=1, padding=0),
+            nn.BatchNorm2d(channels)
+        )
+        self.proj2_conv = DCN_v2_Ref(channels, channels, kernel_size=(3, 3), padding=1)
+        self.proj2_norm = nn.BatchNorm2d(channels)
+
+    def forward(self, feature):
+        # B x C x H x W
+        flipped = feature.flip(-1)  # An auto-copy
+
+        feature = self.proj1(feature)
+        flipped = self.proj2_conv(flipped, feature)
+        flipped = self.proj2_norm(flipped)
+
+        return F.relu(feature + flipped)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/plugins/ppm.py

@@ -0,0 +1,34 @@
+import torch.nn as nn
+from torch.nn import functional as F
+
+
+class PPM(nn.ModuleList):
+    """
+    Pooling pyramid module used in PSPNet
+    Args:
+        pool_scales(tuple(int)): Pooling scales used in pooling Pyramid Module
+        applied on the last feature. default: (1, 2, 3, 6)
+    """
+
+    def __init__(self, pool_scales, in_channels, channels, align_corners=False):
+        super(PPM, self).__init__()
+        self.pool_scales = pool_scales
+        self.align_corners = align_corners
+        self.in_channels = in_channels
+        self.channels = channels
+        for pool_scale in pool_scales:
+            self.append(
+                nn.Sequential(
+                    nn.AdaptiveAvgPool2d(pool_scale),
+                    nn.Conv2d(self.in_channels, self.channels, 1),
+                    nn.BatchNorm2d(self.channels),
+                    nn.ReLU()))
+
+    def forward(self, x):
+        ppm_outs = []
+        for ppm in self:
+            ppm_out = ppm(x)
+            upsampled_ppm_out = F.interpolate(ppm_out, size=x.size()[2:], mode='bilinear',
+                                              align_corners=self.align_corners)
+            ppm_outs.append(upsampled_ppm_out)
+        return ppm_outs

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/plugins/resa_layer.py

@@ -0,0 +1,93 @@
+import math
+import torch
+from torch import nn as nn
+from torch.nn import functional as F
+
+from ...builder import MODELS
+from ..._utils import is_tracing
+
+
+@MODELS.register()
+class RESA(nn.Module):
+    # REcurrent Feature-Shift Aggregator in RESA paper
+
+    def __init__(self, num_channels=128, iteration=5, alpha=2.0, trace_arg=None, os=8):
+        super(RESA, self).__init__()
+        # Different from SCNN, RESA uses bias=False & different convolution layers for each stride,
+        # i.e. 4 * iteration layers vs. 4 layers in SCNN, maybe special init is not needed anymore:
+        # https://github.com/ZJULearning/resa/blob/14b0fea6a1ab4f45d8f9f22fb110c1b3e53cf12e/models/resa.py#L21
+        self.iteration = iteration
+        self.alpha = alpha
+        self.conv_d = nn.ModuleList(nn.Conv2d(num_channels, num_channels, (1, 9), padding=(0, 4), bias=False)
+                                    for _ in range(iteration))
+        self.conv_u = nn.ModuleList(nn.Conv2d(num_channels, num_channels, (1, 9), padding=(0, 4), bias=False)
+                                    for _ in range(iteration))
+        self.conv_r = nn.ModuleList(nn.Conv2d(num_channels, num_channels, (9, 1), padding=(4, 0), bias=False)
+                                    for _ in range(iteration))
+        self.conv_l = nn.ModuleList(nn.Conv2d(num_channels, num_channels, (9, 1), padding=(4, 0), bias=False)
+                                    for _ in range(iteration))
+        self._adjust_initializations(num_channels=num_channels)
+        if trace_arg is not None:  # Pre-compute offsets for a TensorRT supported implementation
+            h = (trace_arg['h'] - 1) // os + 1
+            w = (trace_arg['w'] - 1) // os + 1
+            self.offset_h = []
+            self.offset_w = []
+            for i in range(self.iteration):
+                self.offset_h.append(h // 2 ** (self.iteration - i))
+                self.offset_w.append(w // 2 ** (self.iteration - i))
+
+    def _adjust_initializations(self, num_channels=128):
+        # https://github.com/XingangPan/SCNN/issues/82
+        bound = math.sqrt(2.0 / (num_channels * 9 * 5))
+        for i in self.conv_d:
+            nn.init.uniform_(i.weight, -bound, bound)
+        for i in self.conv_u:
+            nn.init.uniform_(i.weight, -bound, bound)
+        for i in self.conv_r:
+            nn.init.uniform_(i.weight, -bound, bound)
+        for i in self.conv_l:
+            nn.init.uniform_(i.weight, -bound, bound)
+
+    def forward(self, x):
+        y = x
+        h, w = y.shape[-2:]
+        if 2 ** self.iteration > min(h, w):
+            print('Too many iterations for RESA, your image size may be too small.')
+
+        # We do indexing here to avoid extra input parameters at __init__(), with almost none computation overhead.
+        # Also, now it won't block arbitrary shaped input.
+        # However, we still need an alternative to Gather for TensorRT
+        # Down
+        for i in range(self.iteration):
+            if is_tracing():
+                temp = torch.cat([y[:, :, self.offset_h[i]:, :], y[:, :, :self.offset_h[i], :]], dim=-2)
+                y = y.add(self.alpha * F.relu(self.conv_d[i](temp)))
+            else:
+                idx = (torch.arange(h) + h // 2 ** (self.iteration - i)) % h
+                y = y + (self.alpha * F.relu(self.conv_d[i](y[:, :, idx, :])))
+        # Up
+        for i in range(self.iteration):
+            if is_tracing():
+                temp = torch.cat([y[:, :, (h - self.offset_h[i]):, :], y[:, :, :(h - self.offset_h[i]), :]], dim=-2)
+                y = y.add(self.alpha * F.relu(self.conv_u[i](temp)))
+            else:
+                idx = (torch.arange(h) - h // 2 ** (self.iteration - i)) % h
+                y = y + (self.alpha * F.relu(self.conv_u[i](y[:, :, idx, :])))
+        # Right
+        for i in range(self.iteration):
+            if is_tracing():
+                temp = torch.cat([y[:, :, :, self.offset_w[i]:], y[:, :, :, :self.offset_w[i]]], dim=-1)
+                y = y.add(self.alpha * F.relu(self.conv_r[i](temp)))
+            else:
+                idx = (torch.arange(w) + w // 2 ** (self.iteration - i)) % w
+                y = y + (self.alpha * F.relu(self.conv_r[i](y[:, :, :, idx])))
+        # Left
+        for i in range(self.iteration):
+            if is_tracing():
+                temp = torch.cat([y[:, :, :, (w - self.offset_w[i]):], y[:, :, :, :(w - self.offset_w[i])]], dim=-1)
+                y = y.add(self.alpha * F.relu(self.conv_l[i](temp)))
+            else:
+                idx = (torch.arange(w) - w // 2 ** (self.iteration - i)) % w
+                y = y + (self.alpha * F.relu(self.conv_l[i](y[:, :, :, idx])))
+
+        return y

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/plugins/resa_reducer.py

@@ -0,0 +1,24 @@
+from torch import nn as nn
+from torch.nn import functional as F
+
+from ...builder import MODELS
+
+
+@MODELS.register()
+class RESAReducer(nn.Module):
+    # Reduce channel (typically to 128), RESA code use no BN nor ReLU
+
+    def __init__(self, in_channels=512, reduce=128, bn_relu=True):
+        super(RESAReducer, self).__init__()
+        self.bn_relu = bn_relu
+        self.conv1 = nn.Conv2d(in_channels, reduce, 1, bias=False)
+        if self.bn_relu:
+            self.bn1 = nn.BatchNorm2d(reduce)
+
+    def forward(self, x):
+        x = self.conv1(x)
+        if self.bn_relu:
+            x = self.bn1(x)
+            x = F.relu(x)
+
+        return x

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/plugins/se.py

@@ -0,0 +1,32 @@
+from torch import nn as nn
+
+from ..._utils import make_divisible
+
+
+class SELayer(nn.Module):
+    """Squeeze-and-Excitation Module.
+    Args:
+        channels (int): The input (and output) channels of the SE layer.
+        ratio (int): Squeeze ratio in SELayer, the intermediate channel will be
+            ``int(channels/ratio)``. Default: 16.
+    """
+
+    def __init__(self, channels, ratio=16, act=nn.ReLU, scale_act=nn.Sigmoid):
+        super(SELayer, self).__init__()
+
+        self.avgpool = nn.AdaptiveAvgPool2d(1)
+        self.fc1 = nn.Conv2d(in_channels=channels, out_channels=make_divisible(channels // ratio, 8), kernel_size=1,
+                             stride=1)
+        self.fc2 = nn.Conv2d(in_channels=make_divisible(channels // ratio, 8), out_channels=channels, kernel_size=1,
+                             stride=1)
+        self.activation = act()
+        self.scale_activation = scale_act()
+
+    def forward(self, x):
+        out = self.avgpool(x)
+        out = self.fc1(out)
+        out = self.activation(out)
+        out = self.fc2(out)
+        out = self.scale_activation(out)
+
+        return x * out

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/common_models/plugins/spatial_conv.py

@@ -0,0 +1,96 @@
+import math
+import torch
+from torch import nn as nn
+from torch.nn import functional as F
+
+from ...builder import MODELS
+from ..._utils import is_tracing
+
+
+@MODELS.register()
+class SpatialConv(nn.Module):
+    # SCNN
+
+    def __init__(self, num_channels=128):
+        super().__init__()
+        self.conv_d = nn.Conv2d(num_channels, num_channels, (1, 9), padding=(0, 4))
+        self.conv_u = nn.Conv2d(num_channels, num_channels, (1, 9), padding=(0, 4))
+        self.conv_r = nn.Conv2d(num_channels, num_channels, (9, 1), padding=(4, 0))
+        self.conv_l = nn.Conv2d(num_channels, num_channels, (9, 1), padding=(4, 0))
+        self._adjust_initializations(num_channels=num_channels)
+
+    def _adjust_initializations(self, num_channels=128):
+        # https://github.com/XingangPan/SCNN/issues/82
+        bound = math.sqrt(2.0 / (num_channels * 9 * 5))
+        nn.init.uniform_(self.conv_d.weight, -bound, bound)
+        nn.init.uniform_(self.conv_u.weight, -bound, bound)
+        nn.init.uniform_(self.conv_r.weight, -bound, bound)
+        nn.init.uniform_(self.conv_l.weight, -bound, bound)
+
+    def build_slice(self, vertical, reverse, shape):
+        # refer to https://github.com/harryhan618/SCNN_Pytorch/blob/master/model.py
+        slices = []
+        if vertical: # h
+            concat_dim = 2 
+            length = shape[concat_dim]
+            for i in range(length):
+                slices.append(
+                    (slice(None), slice(None), slice(i, i+1), slice(None))
+                )
+        else: # w
+            concat_dim = 3
+            length = shape[concat_dim]
+            for i in range(length):
+                slices.append(
+                    (slice(None), slice(None), slice(None), slice(i, i+1))
+                )
+        if reverse:
+            slices = slices[::-1]
+        return slices, concat_dim
+
+    def non_inplace_forward(self, input):
+        output = input
+        vertical = [True, True, False, False]
+        reverse = [False, True, False, True]
+        convs = [self.conv_d, self.conv_u, self.conv_r, self.conv_l]
+        for ver, rev, conv in zip(vertical, reverse, convs):
+            slices, dim = self.build_slice(ver, rev, input.shape)
+            output_slices = []
+            for idx, s in enumerate(slices):
+                # the condition is to align with the original forward
+                if (rev is False and idx > 0) or (rev is True and idx < len(slices) - 1 and idx > 0):
+                    output_slices.append(
+                        output[s]  + F.relu(conv(output_slices[-1]))
+                    )
+                else:
+                    output_slices.append(output[s])
+            if rev:
+                output_slices = output_slices[::-1]
+            output = torch.cat(output_slices, dim=dim)
+            if ver is False and rev is True:
+                break
+        return output
+
+    def forward(self, input):
+        output = input
+
+        if is_tracing():
+            # PyTorch index+add_ will be ignored in traced graph
+            # Down
+            for i in range(1, output.shape[2]):
+                output[:, :, i:i + 1, :] = output[:, :, i:i + 1, :].add(F.relu(self.conv_d(output[:, :, i - 1:i, :])))
+            # Up
+            for i in range(output.shape[2] - 2, 0, -1):
+                output[:, :, i:i + 1, :] = output[:, :, i:i + 1, :].add(
+                    F.relu(self.conv_u(output[:, :, i + 1:i + 2, :])))
+            # Right
+            for i in range(1, output.shape[3]):
+                output[:, :, :, i:i + 1] = output[:, :, :, i:i + 1].add(F.relu(self.conv_r(output[:, :, :, i - 1:i])))
+            # Left
+            for i in range(output.shape[3] - 2, 0, -1):
+                output[:, :, :, i:i + 1] = output[:, :, :, i:i + 1].add(
+                    F.relu(self.conv_l(output[:, :, :, i + 1:i + 2])))
+        else:
+            output = self.non_inplace_forward(output)
+
+        return output

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/erfnet_encoder.py

@@ -0,0 +1,70 @@
+# modified from utils/models/segmentation/erfnet.py
+# load pretrained weights during initialization of encoder
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from .common_models import non_bottleneck_1d
+from .builder import MODELS
+
+
+class DownsamplerBlock(nn.Module):
+    def __init__(self, ninput, noutput):
+        super().__init__()
+        self.conv = nn.Conv2d(ninput, noutput-ninput, (3, 3), stride=2, padding=1, bias=True)
+        self.pool = nn.MaxPool2d(2, stride=2)
+        self.bn = nn.BatchNorm2d(noutput, eps=1e-3)
+
+    def forward(self, input):
+        output = torch.cat([self.conv(input), self.pool(input)], 1)
+        output = self.bn(output)
+        return F.relu(output)
+
+@MODELS.register()
+class ERFNetEncoder(nn.Module):
+    def __init__(self, num_classes, dropout_1=0.03, dropout_2=0.3, pretrained_weights=None):
+        super().__init__()
+        self.initial_block = DownsamplerBlock(3, 16)
+        self.layers = nn.ModuleList()
+        self.layers.append(DownsamplerBlock(16, 64))
+        for x in range(0, 5):  # 5 times
+            self.layers.append(non_bottleneck_1d(64, dropout_1, 1))
+        self.layers.append(DownsamplerBlock(64, 128))
+        for x in range(0, 2):  # 2 times
+            self.layers.append(non_bottleneck_1d(128, dropout_2, 2))
+            self.layers.append(non_bottleneck_1d(128, dropout_2, 4))
+            self.layers.append(non_bottleneck_1d(128, dropout_2, 8))
+            self.layers.append(non_bottleneck_1d(128, dropout_2, 16))
+
+        # need to initialize the weights
+        if pretrained_weights is not None:
+            self._load_encoder_weights(pretrained_weights) # Load ImageNet pre-trained weights
+        else:
+            self._init_weights() # initialize random weights
+
+    def _init_weights(self):
+        pass
+
+    def _load_encoder_weights(self, pretrained_weights):
+        # load weights from given file path
+        try:
+            saved_weights = torch.load(pretrained_weights)['state_dict']
+        except FileNotFoundError:
+            raise FileNotFoundError('pretrained_weights is not there! '
+                                    'Please set pretrained_weights=None if you are only testing.')
+        original_weights = self.state_dict()
+        for key in saved_weights.keys():
+            my_key = key.replace('module.features.', '')
+            if my_key in original_weights.keys():
+                original_weights[my_key] = saved_weights[key]
+        self.load_state_dict(original_weights)
+
+    def forward(self, input):
+        output = self.initial_block(input)
+        for layer in self.layers:
+            output = layer(output)
+
+        return output
+
+

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/lane_detection/__init__.py

@@ -0,0 +1,5 @@
+from .lstr import LSTR, cubic_curve_with_projection
+from .resa import RESA_Net
+from .bezier_lane_net import BezierLaneNet
+from . import utils
+from .laneatt import LaneAtt

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/lane_detection/bezier_base.py

@@ -0,0 +1,95 @@
+import torch
+import numpy as np
+
+from .utils import lane_pruning
+from ...curve_utils import BezierCurve
+
+
+class BezierBaseNet(torch.nn.Module):
+    def __init__(self, thresh=0.5, local_maximum_window_size=9):
+        super().__init__()
+        self.thresh = thresh
+        self.local_maximum_window_size = local_maximum_window_size
+
+    def forward(self, *args, **kwargs):
+        raise NotImplementedError
+
+    @staticmethod
+    def bezier_to_coordinates(control_points, existence, resize_shape, dataset, bezier_curve, ppl=56, gap=10):
+        # control_points: L x N x 2
+        H, W = resize_shape
+        cps_of_lanes = []
+        for flag, cp in zip(existence, control_points):
+            if flag:
+                cps_of_lanes.append(cp.tolist())
+        coordinates = []
+        for cps_of_lane in cps_of_lanes:
+            bezier_curve.assign_control_points(cps_of_lane)
+            if dataset == 'tusimple':
+                # Find x for TuSimple's fixed y eval positions (suboptimal)
+                bezier_threshold = 5.0 / H
+                h_samples = np.array([1.0 - (ppl - i) * gap / H for i in range(ppl)], dtype=np.float32)
+                sampled_points = bezier_curve.quick_sample_point(image_size=None)
+                temp = []
+                dis = np.abs(np.expand_dims(h_samples, -1) - sampled_points[:, 1])
+                idx = np.argmin(dis, axis=-1)
+                for i in range(ppl):
+                    h = H - (ppl - i) * gap
+                    if dis[i][idx[i]] > bezier_threshold or sampled_points[idx[i]][0] > 1 or sampled_points[idx[i]][0] < 0:
+                        temp.append([-2, h])
+                    else:
+                        temp.append([sampled_points[idx[i]][0] * W, h])
+                coordinates.append(temp)
+            elif dataset in ['culane', 'llamas']:
+                temp = bezier_curve.quick_sample_point(image_size=None)
+                temp[:, 0] = temp[:, 0] * W
+                temp[:, 1] = temp[:, 1] * H
+                coordinates.append(temp.tolist())
+            else:
+                raise ValueError
+
+        return coordinates
+
+    @torch.no_grad()
+    def inference(self, inputs, input_sizes, gap, ppl, dataset, max_lane=0, forward=True, return_cps=False, n=50):
+        outputs = self.forward(inputs) if forward else inputs  # Support no forwarding inside this function
+        existence_conf = outputs['logits'].sigmoid()
+        existence = existence_conf > self.thresh
+
+        # Test local maxima
+        if self.local_maximum_window_size > 0:
+            _, max_indices = torch.nn.functional.max_pool1d(existence_conf.unsqueeze(1),
+                                                            kernel_size=self.local_maximum_window_size, stride=1,
+                                                            padding=(self.local_maximum_window_size - 1) // 2,
+                                                            return_indices=True)
+            max_indices = max_indices.squeeze(1)  # B x Q
+            indices = torch.arange(0, existence_conf.shape[1],
+                                   dtype=existence_conf.dtype,
+                                   device=existence_conf.device).unsqueeze(0).expand_as(max_indices)
+            local_maxima = max_indices == indices
+            existence *= local_maxima
+
+        control_points = outputs['curves']
+        if max_lane != 0:  # Lane max number prior for testing
+            existence, _ = lane_pruning(existence, existence_conf, max_lane=max_lane)
+
+        if return_cps:
+            image_size = torch.tensor([input_sizes[1][1], input_sizes[1][0]],
+                                      dtype=torch.float32, device=control_points.device)
+            cps = control_points * image_size
+            cps = [cps[i][existence[i]].cpu().numpy() for i in range(existence.shape[0])]
+
+        existence = existence.cpu().numpy()
+        control_points = control_points.cpu().numpy()
+        H, _ = input_sizes[1]
+        b = BezierCurve(order=3, num_sample_points=H if dataset == 'tusimple' else n)
+
+        lane_coordinates = []
+        for j in range(existence.shape[0]):
+            lane_coordinates.append(self.bezier_to_coordinates(control_points=control_points[j], existence=existence[j],
+                                                               resize_shape=input_sizes[1], dataset=dataset,
+                                                               bezier_curve=b, gap=gap, ppl=ppl))
+        if return_cps:
+            return cps, lane_coordinates
+        else:
+            return lane_coordinates

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/lane_detection/bezier_lane_net.py

@@ -0,0 +1,74 @@
+import torch
+import torch.nn as nn
+if torch.__version__ >= '1.6.0':
+    from torch.cuda.amp import autocast
+else:
+    from utils.torch_amp_dummy import autocast
+
+from .bezier_base import BezierBaseNet
+from ..builder import MODELS
+
+
+@MODELS.register()
+class BezierLaneNet(BezierBaseNet):
+    # Curve regression network, similar design as simple object detection (e.g. FCOS)
+    def __init__(self,
+                 backbone_cfg,
+                 reducer_cfg,
+                 dilated_blocks_cfg,
+                 feature_fusion_cfg,
+                 head_cfg,
+                 aux_seg_head_cfg,
+                 image_height=360,
+                 num_regression_parameters=8,
+                 thresh=0.5,
+                 local_maximum_window_size=9):
+        super(BezierLaneNet, self).__init__(thresh, local_maximum_window_size)
+        global_stride = 16
+        branch_channels = 256
+
+        self.backbone = MODELS.from_dict(backbone_cfg)
+        self.reducer = MODELS.from_dict(reducer_cfg)
+        self.dilated_blocks = MODELS.from_dict(dilated_blocks_cfg)
+        self.simple_flip_2d = MODELS.from_dict(feature_fusion_cfg)  # Name kept for legacy weights
+        self.aggregator = nn.AvgPool2d(kernel_size=((image_height - 1) // global_stride + 1, 1), stride=1, padding=0)
+        self.regression_head = MODELS.from_dict(head_cfg)  # Name kept for legacy weights
+        self.proj_classification = nn.Conv1d(branch_channels, 1, kernel_size=1, bias=True, padding=0)
+        self.proj_regression = nn.Conv1d(branch_channels, num_regression_parameters,
+                                         kernel_size=1, bias=True, padding=0)
+        self.segmentation_head = MODELS.from_dict(aux_seg_head_cfg)
+
+    def forward(self, x):
+        # Return shape: B x Q, B x Q x N x 2
+        x = self.backbone(x)
+        if isinstance(x, dict):
+            x = x['out']
+
+        if self.reducer is not None:
+            x = self.reducer(x)
+
+        # Segmentation task
+        if self.segmentation_head is not None:
+            segmentations = self.segmentation_head(x)
+        else:
+            segmentations = None
+
+        if self.dilated_blocks is not None:
+            x = self.dilated_blocks(x)
+
+        with autocast(False):  # TODO: Support fp16 like mmcv
+            x = self.simple_flip_2d(x.float())
+        x = self.aggregator(x)[:, :, 0, :]
+
+        x = self.regression_head(x)
+        logits = self.proj_classification(x).squeeze(1)
+        curves = self.proj_regression(x)
+
+        return {'logits': logits,
+                'curves': curves.permute(0, 2, 1).reshape(curves.shape[0], -1, curves.shape[-2] // 2, 2).contiguous(),
+                'segmentations': segmentations}
+
+    def eval(self, profiling=False):
+        super().eval()
+        if profiling:
+            self.segmentation_head = None

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/lane_detection/laneatt.py

@@ -0,0 +1,342 @@
+# Refactored from lucastabelini/LaneATT
+# Diffs:
+# 1. we changed lane rep to 74 numbers (start, len, 72 offsets)
+# 2. we use a cleaner line nms dynamically loaded (input only 74 numbers, not 77)
+# 3. we removed unnecessary inputs & outputs in post-processing funcs
+# 4. we removed B-Spline interpolation post-processing to provide fair comparisons (test 74.88 for resnet18-CULane in official code)
+# After these simplifications & refactors, our testing procedure seems even better than the original
+
+import math
+import torch
+import numpy as np
+import torch.nn as nn
+from torch.nn import functional as F
+from scipy.interpolate import splprep, splev
+from .._utils import is_tracing
+
+try:
+    from ...csrc.apis import line_nms
+    print('Successfully complied line nms for LaneATT.')
+except:
+    from ...common import warnings
+    warnings.warn('Can\'t complie line nms op for LaneATT. Set verbose=True for load in /utils/csrc/apis.py L9 for details.')
+from ..builder import MODELS
+
+
+@MODELS.register()
+class LaneAtt(nn.Module):
+    # Anchor angles, same ones used in Line-CNN
+    left_angles = [72., 60., 49., 39., 30., 22.]
+    right_angles = [108., 120., 131., 141., 150., 158.]
+    bottom_angles = [165., 150., 141., 131., 120., 108., 100., 90., 80., 72., 60., 49., 39., 30., 15.]
+
+    def __init__(self,
+                 backbone_cfg,
+                 backbone_channels,
+                 backbone_os,
+                 num_points=72,
+                 img_w=640,
+                 img_h=360,
+                 topk_anchors=None,
+                 anchor_freq_path=None,
+                 anchor_feat_channels=None,
+                 conf_thres=None,
+                 nms_thres=0,
+                 nms_topk=3000,
+                 trace_arg=None):
+        super().__init__()
+        self.backbone = MODELS.from_dict(backbone_cfg)
+        self.backbone_channels = backbone_channels
+        self.stride = backbone_os
+        self.num_strips = num_points - 1
+        self.num_offsets = num_points
+        self.img_h = img_h
+        self.img_w = img_w
+        self.featmap_h = img_h // self.stride
+        self.featmap_w = img_w // self.stride
+        self.anchor_ys = torch.linspace(1, 0, steps=self.num_offsets, dtype=torch.float32)
+        self.anchor_cut_ys = torch.linspace(1, 0, steps=self.featmap_h, dtype=torch.float32)
+        self.anchor_feat_channels = anchor_feat_channels
+        # nms config
+        self.conf_thres = conf_thres
+        self.nms_thres = nms_thres
+        self.nms_topk = nms_topk
+
+        if trace_arg is not None:  # Pre-compute
+            attention_matrix = torch.eye(topk_anchors).repeat(trace_arg['bs'], 1, 1)
+            self.pre_non_diag_inds = torch.nonzero(attention_matrix == 0., as_tuple=False)
+
+        # generate anchors
+        self.anchors, self.anchors_cut = self.generate_anchors(lateral_n=72, bottom_n=128)
+        # Filter masks if `anchors_freq_path` is provided
+        if anchor_freq_path is not None:
+            anchors_mask = torch.load(anchor_freq_path).cpu()
+            assert topk_anchors is not None, 'topk_anchors cannot be None'
+            idx = torch.argsort(anchors_mask, descending=True)[: topk_anchors]
+            self.anchors = self.anchors[idx]
+            self.anchors_cut = self.anchors_cut[idx]
+
+        # pre compute indices for the anchor pooling
+        self.cut_zs, self.cut_ys, self.cut_xs, self.invalid_mask = self.compute_anchor_cut_indices(
+            self.anchor_feat_channels, self.featmap_w, self.featmap_h)
+
+        # Setup ans initialize layers
+        self.conv1 = nn.Conv2d(self.backbone_channels, self.anchor_feat_channels, kernel_size=1)
+        self.cls_layer = nn.Linear(2 * self.anchor_feat_channels * self.featmap_h, 2)
+        self.reg_layer = nn.Linear(2 * self.anchor_feat_channels * self.featmap_h, self.num_offsets + 1)
+        self.attention_layer = nn.Linear(self.anchor_feat_channels * self.featmap_h, len(self.anchors) - 1)
+        self.initialize_layer(self.attention_layer)
+        self.initialize_layer(self.conv1)
+        self.initialize_layer(self.cls_layer)
+        self.initialize_layer(self.reg_layer)
+
+    def generate_anchors(self, lateral_n, bottom_n):
+        left_anchors, left_cut = self.generate_side_anchors(self.left_angles, x=0., nb_origins=lateral_n)
+        right_anchors, right_cut = self.generate_side_anchors(self.right_angles, x=1., nb_origins=lateral_n)
+        bottom_anchors, bottom_cut = self.generate_side_anchors(self.bottom_angles, y=1., nb_origins=bottom_n)
+
+        return torch.cat([left_anchors, bottom_anchors, right_anchors]), \
+            torch.cat([left_cut, bottom_cut, right_cut])
+
+    def generate_side_anchors(self, angles, nb_origins, x=None, y=None):
+        if x is None and y is not None:
+            starts = [(x, y) for x in np.linspace(1., 0., num=nb_origins)]
+        elif x is not None and y is None:
+            starts = [(x, y) for y in np.linspace(1., 0., num=nb_origins)]
+        else:
+            raise Exception('Please define exactly one of `x` or `y` (not neither nor both)')
+
+        n_anchors = nb_origins * len(angles)
+
+        # each row, first for x and second for y:
+        # 2 scores, 1 start_y, start_x, 1 length, num_points coordinates
+        # score[0] = negative prob, score[0] = positive prob
+        anchors = torch.zeros((n_anchors, 2 + self.num_offsets))
+        anchors_cut = torch.zeros((n_anchors, 2 + self.featmap_h))
+        for i, start in enumerate(starts):
+            for j, angle in enumerate(angles):
+                k = i * len(angles) + j
+                anchors[k] = self.generate_anchor(start, angle)
+                anchors_cut[k] = self.generate_anchor(start, angle, cut=True)
+
+        return anchors, anchors_cut
+
+    def generate_anchor(self, start, angle, cut=False):
+        if cut:
+            anchor_ys = self.anchor_cut_ys
+            anchor = torch.zeros(2 + self.featmap_h)
+        else:
+            anchor_ys = self.anchor_ys
+            anchor = torch.zeros(2 + self.num_offsets)
+        angle = angle * math.pi / 180.  # degrees to radians
+        start_x, start_y = start
+        anchor[0] = 1 - start_y  # using left bottom as the (0, 0) of the axis ?
+        anchor[1] = start_x
+        anchor[2:] = (start_x + (1 - anchor_ys - 1 + start_y) / math.tan(angle)) * self.img_w
+
+        return anchor
+
+    def compute_anchor_cut_indices(self, num_channels, feat_w, feat_h):
+        # definitions
+        num_proposal = len(self.anchors_cut)
+
+        # indexing
+        # num_anchors x feat_h
+        unclamped_xs = torch.flip((self.anchors_cut[:, 2:] / self.stride).round().long(), dims=(1, ))
+        unclamped_xs = unclamped_xs[..., None]
+        # num_channels x num_anchors x feat_h --> num_channels * num_anchors * feat_h x 1
+        unclamped_xs = torch.repeat_interleave(unclamped_xs, num_channels, dim=0).reshape(-1, 1)
+        cut_xs = torch.clamp(unclamped_xs, 0, feat_w - 1)
+        unclamped_xs = unclamped_xs.reshape(num_proposal, num_channels, feat_h, 1)
+        invalid_mask = (unclamped_xs < 0) | (unclamped_xs > feat_w)
+        cut_ys = torch.arange(0, feat_h)
+        cut_ys = cut_ys.repeat(num_channels * num_proposal)[:, None].reshape(num_proposal, num_channels, feat_h)
+        cut_ys = cut_ys.reshape(-1, 1)
+        cut_zs = torch.arange(num_channels).repeat_interleave(feat_h).repeat(num_proposal)[:, None]
+
+        return cut_zs, cut_ys, cut_xs, invalid_mask
+
+    def cut_anchor_features(self, features):
+        # definitions
+        batch_size = features.shape[0]
+        n_proposals = len(self.anchors)
+        n_fmaps = features.shape[1]
+        batch_anchor_features = torch.zeros((batch_size, n_proposals, n_fmaps, self.featmap_h, 1),
+                                            device=features.device)
+
+        # actual cutting
+        for batch_idx, img_features in enumerate(features):
+            rois = img_features[self.cut_zs, self.cut_ys, self.cut_xs].view(n_proposals, n_fmaps, self.featmap_h, 1)
+            rois[self.invalid_mask] = 0
+            batch_anchor_features[batch_idx] = rois
+
+        return batch_anchor_features
+
+    @staticmethod
+    def initialize_layer(layer):
+        if isinstance(layer, (nn.Conv2d, nn.Linear)):
+            torch.nn.init.normal_(layer.weight, mean=0., std=0.001)
+            if layer.bias is not None:
+                torch.nn.init.constant_(layer.bias, 0)
+
+    def cuda(self, device=None):
+        cuda_self = super().cuda(device)
+        cuda_self.anchors = cuda_self.anchors.cuda(device)
+        cuda_self.anchor_ys = cuda_self.anchor_ys.cuda(device)
+        cuda_self.cut_zs = cuda_self.cut_zs.cuda(device)
+        cuda_self.cut_ys = cuda_self.cut_ys.cuda(device)
+        cuda_self.cut_xs = cuda_self.cut_xs.cuda(device)
+        cuda_self.invalid_mask = cuda_self.invalid_mask.cuda(device)
+        return cuda_self
+
+    def to(self, *args, **kwargs):
+        device_self = super().to(*args, **kwargs)
+        device_self.anchors = device_self.anchors.to(*args, **kwargs)
+        device_self.anchor_ys = device_self.anchor_ys.to(*args, **kwargs)
+        device_self.cut_zs = device_self.cut_zs.to(*args, **kwargs)
+        device_self.cut_ys = device_self.cut_ys.to(*args, **kwargs)
+        device_self.cut_xs = device_self.cut_xs.to(*args, **kwargs)
+        device_self.invalid_mask = device_self.invalid_mask.to(*args, **kwargs)
+        return device_self
+
+    def forward(self, x):
+        batch_features = self.backbone(x)['out']
+
+        batch_features = self.conv1(batch_features)
+        # batchsize x num_proposals x anchor_feat_channels x featmap_h x 1
+        batch_anchor_features = self.cut_anchor_features(batch_features)
+
+        # join proposals from all images into a single proposals features batch
+        batch_anchor_features = batch_anchor_features.view(-1, self.anchor_feat_channels * self.featmap_h)
+        # add attention features
+        softmax = nn.Softmax(dim=1)
+        scores = self.attention_layer(batch_anchor_features)
+        attention = softmax(scores).reshape(x.shape[0], len(self.anchors), -1)
+        attention_matrix = torch.eye(attention.shape[1], device=x.device).repeat(x.shape[0], 1, 1)
+        if is_tracing():
+            # Use pre-computed nonzero results
+            non_diag_inds = self.pre_non_diag_inds.to(attention_matrix.device)
+        else:
+            non_diag_inds = torch.nonzero(attention_matrix == 0., as_tuple=False)
+        attention_matrix[:] = 0
+        attention_matrix[non_diag_inds[:, 0], non_diag_inds[:, 1], non_diag_inds[:, 2]] = attention.flatten()
+        batch_anchor_features = batch_anchor_features.reshape(x.shape[0], len(self.anchors), -1)
+        attention_features = torch.bmm(torch.transpose(batch_anchor_features, 1, 2),
+                                       torch.transpose(attention_matrix, 1, 2)).transpose(1, 2)
+        attention_features = attention_features.reshape(-1, self.anchor_feat_channels * self.featmap_h)
+        batch_anchor_features = batch_anchor_features.reshape(-1, self.anchor_feat_channels * self.featmap_h)
+        batch_anchor_features = torch.cat((attention_features, batch_anchor_features), dim=1)
+
+        # predict
+        cls_logits = self.cls_layer(batch_anchor_features)
+        reg = self.reg_layer(batch_anchor_features)
+
+        # Undo joining
+        cls_logits = cls_logits.reshape(x.shape[0], -1, cls_logits.shape[1])
+        reg = reg.reshape(x.shape[0], -1, reg.shape[1])
+
+        # Add offset to anchors
+        reg_proposals = torch.zeros((*cls_logits.shape[:2], self.num_offsets + 2), device=x.device)
+        reg_proposals += self.anchors
+        reg_proposals[:, :, 1:] += reg
+
+        return {
+            'offsets': reg_proposals[..., 2:],  # B x M x 72
+            'starts': reg_proposals[..., 0],  # B x M x 1
+            'lengths': reg_proposals[..., 1],  # B x M x 1
+            'logits': cls_logits  # B x M x 2
+        }
+
+    @torch.no_grad()
+    def inference(self, inputs, input_sizes, forward=True, *args, **kwargs):
+        outputs = self.forward(inputs) if forward else inputs  # Support no forwarding inside this function
+        to_tusimple = True if args[1] == "tusimple" else False
+        batch_regs = self.nms(outputs)
+        # the number of lanes is 1 ???
+        decoded = []
+        for regs in batch_regs:
+            regs[:, 1] = torch.round(regs[:, 1])  # length
+            if regs.shape[0] == 0:
+                decoded.append([])
+                continue
+            pred = self.proposals_to_pred(regs, input_sizes[1], to_tusimple)
+            decoded.append(pred)
+        return decoded
+
+    @torch.no_grad()
+    def nms(self, batch_proposals):
+        proposals_list = []
+        for i in range(len(batch_proposals['logits'])):
+            scores = F.softmax(batch_proposals['logits'][i], dim=1)[:, 1]
+            regs = torch.cat([batch_proposals['starts'][i][..., None],
+                              batch_proposals['lengths'][i][..., None],
+                              batch_proposals['offsets'][i]], dim=1)
+            if self.conf_thres is not None:
+                # apply confidence threshold
+                above_threshold = scores > self.conf_thres
+                regs = regs[above_threshold]
+                scores = scores[above_threshold]
+            if regs.shape[0] == 0:
+                proposals_list.append(regs[[]])
+                continue
+            keep, num_to_keep, _ = line_nms(regs, scores, self.nms_thres, self.nms_topk)
+            keep = keep[:num_to_keep]
+            regs = regs[keep]
+            proposals_list.append(regs)
+
+        return proposals_list
+
+    def proposals_to_pred(self, proposals, image_size, to_tusimple=False):
+        self.anchor_ys = self.anchor_ys.to(proposals.device)
+        lanes = []
+        for lane in proposals:
+            lane_xs = lane[2:] / self.img_w
+
+            # start end length in 0-72
+            start = int(round(lane[0].item() * self.num_strips))
+            length = int(round(lane[1].item()))
+            end = start + length - 1
+            end = min(end, len(self.anchor_ys) - 1)
+
+            # end = label_end
+            # if the proposal does not start at the bottom of the image,
+            # extend its proposal until the x is outside the image
+            mask = ~((((lane_xs[:start] >= 0.) &
+                       (lane_xs[:start] <= 1.)).cpu().numpy()[::-1].cumprod()[::-1]).astype(np.bool))
+            lane_xs[end + 1:] = -2
+            lane_xs[:start][mask] = -2
+            lane_ys = self.anchor_ys[lane_xs >= 0]
+            lane_xs = lane_xs[lane_xs >= 0]
+            lane_xs = lane_xs.flip(0)
+            lane_ys = lane_ys.flip(0)
+            if len(lane_xs) <= 1:
+                continue
+            points = torch.stack((lane_xs.reshape(-1, 1), lane_ys.reshape(-1, 1)), dim=1).squeeze(2)
+            points = points.cpu().numpy()
+            lane_coords = []
+            for i in range(points.shape[0]):
+                lane_coords.append([points[i, 0] * float(image_size[1]), points[i, 1] * float(image_size[0])])
+            if to_tusimple:
+                lanes.append(self.convert_to_tusimple(lane_coords))
+            else:
+                lanes.append(lane_coords)
+
+        return lanes
+
+    def convert_to_tusimple(self, points, n=200, bezier_threshold=5):
+        """Spline interpolation of a lane. Used on the predictions"""
+        x = [x for x, _ in points]
+        y = [y for _, y in points]
+        tck, _ = splprep([x, y], s=0, t=n, k=min(3, len(points) - 1))
+        u = np.linspace(0., 1., n)
+        rep_points = np.array(splev(u, tck)).T
+        h_samples = [(160 + y * 10) for y in range(56)]
+        temp = []
+        for h_sample in h_samples:
+            dis = np.abs(h_sample - rep_points[:, 1])
+            idx = np.argmin(dis)
+            if dis[idx] > bezier_threshold:
+                temp.append([-2, h_sample])
+            else:
+                temp.append([round(rep_points[:, 0][idx], 3), h_sample])
+        return temp

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/lane_detection/lstr.py

@@ -0,0 +1,176 @@
+# Adapted from liuruijin17/LSTR
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from .utils import lane_pruning
+from ..transformer import build_transformer, build_position_encoding
+from ..mlp import MLP
+from .._utils import is_tracing
+from ..builder import MODELS
+
+
+def cubic_curve_with_projection(coefficients, y):
+    # The cubic curve model from LSTR (considers projection to image plane)
+    # Return x coordinates
+    # coefficients: [d1, d2, ..., 6]
+    # 6 coefficients: [k", f", m", n", b", b''']
+    # y: [d1, d2, ..., N]
+    y = y.permute(-1, *[i for i in range(len(y.shape) - 1)])  # -> [N, d1, d2, ...]
+    x = coefficients[..., 0] / (y - coefficients[..., 1]) ** 2 \
+        + coefficients[..., 2] / (y - coefficients[..., 1]) \
+        + coefficients[..., 3] \
+        + coefficients[..., 4] * y \
+        - coefficients[..., 5]
+
+    return x.permute(*[i + 1 for i in range(len(x.shape) - 1)], 0)  # [d1, d2, ... , N]
+
+
+@MODELS.register()
+class LSTR(nn.Module):
+    def __init__(self,
+                 expansion=1,  # Expansion rate (1x for TuSimple & 2x for CULane)
+                 num_queries=7,  # Maximum number of lanes
+                 aux_loss=True,  # Important for transformer-based methods
+                 pos_type='sine',
+                 drop_out=0.1,
+                 num_heads=2,
+                 enc_layers=2,
+                 dec_layers=2,
+                 pre_norm=False,
+                 return_intermediate=True,
+                 lsp_dim=8,
+                 mlp_layers=3,
+                 backbone_cfg=None,
+                 thresh=0.95,
+                 trace_arg=None):
+        super().__init__()
+
+        self.thresh = thresh
+        self.backbone = MODELS.from_dict(backbone_cfg)
+        hidden_dim = 32 * expansion
+        self.aux_loss = aux_loss
+        self.position_embedding = build_position_encoding(hidden_dim=hidden_dim, position_embedding=pos_type)
+        if trace_arg is not None:  # Pre-compute embeddings
+            trace_arg['h'] = (trace_arg['h'] - 1) // 32 + 1
+            trace_arg['w'] = (trace_arg['w'] - 1) // 32 + 1
+            x = torch.zeros((trace_arg['bs'], trace_arg['h'], trace_arg['w']), dtype=torch.bool)
+            y = torch.zeros((trace_arg['bs'], 128 * expansion, trace_arg['h'], trace_arg['w']), dtype=torch.float32)
+            self.pos = torch.nn.Parameter(data=self.position_embedding(y, x), requires_grad=False)
+        self.query_embed = nn.Embedding(num_queries, hidden_dim)
+        self.input_proj = nn.Conv2d(128 * expansion, hidden_dim, kernel_size=1)  # Same channel as layer4
+
+        self.transformer = build_transformer(hidden_dim=hidden_dim,
+                                             dropout=drop_out,
+                                             nheads=num_heads,
+                                             dim_feedforward=128 * expansion,
+                                             enc_layers=enc_layers,
+                                             dec_layers=dec_layers,
+                                             pre_norm=pre_norm,
+                                             return_intermediate_dec=return_intermediate)
+
+        # Original LSTR: 3 classes + CE (softmax), we use 2
+        self.class_embed = nn.Linear(hidden_dim, 2)
+
+        self.specific_embed = MLP(hidden_dim, hidden_dim, lsp_dim - 4, mlp_layers)  # Specific for each lane
+        self.shared_embed = MLP(hidden_dim, hidden_dim, 4, mlp_layers)  # 4 shared curve coefficients
+
+    def forward(self, images, padding_masks=None):
+        # images: B x C x H x W
+        # padding_masks: B x H x W (0 or 1 -> ignored)
+        p = self.backbone(images)['out']
+
+        # Padding mask (for paddings added in transforms)
+        if is_tracing():
+            pos = self.pos
+        else:
+            if padding_masks is None:  # Make things easier for testing (assume no padding)
+                padding_masks = torch.zeros((p.shape[0], p.shape[2], p.shape[3]), dtype=torch.bool, device=p.device)
+            else:
+                padding_masks = F.interpolate(padding_masks[None].float(), size=p.shape[-2:]).to(torch.bool)[0]
+            pos = self.position_embedding(p, padding_masks)
+
+        hs, _ = self.transformer(self.input_proj(p), padding_masks, self.query_embed.weight, pos)
+        output_class = self.class_embed(hs)
+        output_specific = self.specific_embed(hs)
+        output_shared = self.shared_embed(hs)
+        output_shared = torch.mean(output_shared, dim=-2, keepdim=True)  # Why not take mean on input and simply expand?
+        output_shared = output_shared.repeat(1, 1, output_specific.shape[2], 1)
+
+        # Keep this for consistency with official LSTR: [upper, lower, k", f", m", n", b", b''']
+        output_curve = torch.cat([output_specific[:, :, :, :2],
+                                  output_shared, output_specific[:, :, :, 2:]], dim=-1)
+
+        out = {'logits': output_class[-1], 'curves': output_curve[-1]}  # Last layer result
+        if self.aux_loss:
+            out['aux'] = self._set_aux_loss(output_class, output_curve)  # All intermediate results
+
+        return out
+
+    def eval(self, profiling=False):
+        super().eval()
+        if profiling:
+            self.aux_loss = False
+            self.transformer.decoder.return_intermediate = False
+
+    @torch.no_grad()
+    def inference(self, inputs, input_sizes, gap, ppl, dataset, max_lane=0, forward=True, **kwargs):
+        outputs = self.forward(inputs) if forward else inputs  # Support no forwarding inside this function
+        existence_conf = outputs['logits'].softmax(dim=-1)[..., 1]
+        existence = existence_conf > self.thresh
+        if max_lane != 0:  # Lane max number prior for testing
+            existence, _ = lane_pruning(existence, existence_conf, max_lane=max_lane)
+
+        existence = existence.cpu().numpy()
+        # Get coordinates for lanes
+        lane_coordinates = []
+        for j in range(existence.shape[0]):
+            lane_coordinates.append(self.coefficients_to_coordinates(outputs['curves'][j, :, 2:], existence[j],
+                                    resize_shape=input_sizes[1], dataset=dataset, ppl=ppl,
+                                    gap=gap, curve_function=cubic_curve_with_projection,
+                                    upper_bound=outputs['curves'][j, :, 0],
+                                    lower_bound=outputs['curves'][j, :, 1]))
+
+        return lane_coordinates
+
+    @staticmethod
+    def coefficients_to_coordinates(coefficients, existence, resize_shape, dataset, ppl, gap, curve_function,
+                                    upper_bound, lower_bound):
+        # For methods that predict coefficients of polynomials,
+        # works with normalized coordinates (in range 0.0 ~ 1.0).
+        # Restricted to single image to align with other methods' codes
+        H, W = resize_shape
+        if dataset == 'tusimple':  # Annotation start at 10 pixel away from bottom
+            y = torch.tensor([1.0 - (ppl - i) * gap / H for i in range(ppl)],
+                             dtype=coefficients.dtype, device=coefficients.device)
+        elif dataset in ['culane', 'llamas']:  # Annotation start at bottom
+            y = torch.tensor([1.0 - i * gap / H for i in range(ppl)],
+                             dtype=coefficients.dtype, device=coefficients.device)
+        else:
+            raise ValueError
+        coords = curve_function(coefficients=coefficients, y=y.unsqueeze(0).expand(coefficients.shape[0], -1))
+
+        # Delete outside points according to predicted upper & lower boundaries
+        coordinates = []
+        for i in range(existence.shape[0]):
+            if existence[i]:
+                # Note that in image coordinate system, (0, 0) is the top-left corner
+                valid_points = (coords[i] >= 0) * (coords[i] <= 1) * (y < lower_bound[i]) * (y > upper_bound[i])
+                if valid_points.sum() < 2:  # Same post-processing technique as segmentation methods
+                    continue
+                if dataset == 'tusimple':  # Invalid sample points need to be included as negative value, e.g. -2
+                    coordinates.append([[(coords[i][j] * W).item(), H - (ppl - j) * gap]
+                                        if valid_points[j] else [-2, H - (ppl - j) * gap] for j in range(ppl)])
+                elif dataset in ['culane', 'llamas']:
+                    coordinates.append([[(coords[i][j] * W).item(), H - j * gap]
+                                        for j in range(ppl) if valid_points[j]])
+                else:
+                    raise ValueError
+
+        return coordinates
+
+    @torch.jit.unused
+    def _set_aux_loss(self, output_class, output_curve):
+        # this is a workaround to make torchscript happy, as torchscript
+        # doesn't support dictionary with non-homogeneous values, such
+        # as a dict having both a Tensor and a list.
+        return [{'logits': a, 'curves': b} for a, b in zip(output_class[:-1], output_curve[:-1])]

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/lane_detection/resa.py

@@ -0,0 +1,50 @@
+# Adapted from ZJULearning/resa
+# Better to use a decoupled implementation,
+# costs more codes, but clear.
+# Diff from RESA official code:
+# 1. we use BN+ReLU in channel reducer
+# 2. we always use the BUSD decoder in the paper (official code does not use BUSD in CULane)
+# 3. we always use 5 RESA iterations (4 in official code)
+# 4. we use a higher capacity lane existence classifier (same as ERFNet/ENet baseline)
+# 5. we use the SCNN sqrt(5) init trick for RESA, which
+# 5.1. enables fewer warmup steps
+# 5.2. combined with 4, produces slightly better performance
+# 6. we do not use horizontal flip or cutting height in loading, in which
+# 6.1. flip does not help performance (at least on the val set)
+# 6.2. w.o. cutting height trick probably is the main reason for our lower performance, but we can't use it since
+# other pytorch-auto-drive models do not use it.
+import torch.nn as nn
+
+from ..builder import MODELS
+
+
+@MODELS.register()
+class RESA_Net(nn.Module):
+    def __init__(self,
+                 backbone_cfg,
+                 reducer_cfg,
+                 spatial_conv_cfg,
+                 classifier_cfg,
+                 lane_classifier_cfg,
+                 trace_arg=None):
+        super().__init__()
+        self.backbone = MODELS.from_dict(backbone_cfg)
+        # self.channel_reducer = RESAReducer(in_channels=in_channels, reduce=channel_reduce, bn_relu=False)
+        self.channel_reducer = MODELS.from_dict(reducer_cfg)
+        self.spatial_conv = MODELS.from_dict(spatial_conv_cfg, trace_arg=trace_arg)
+        self.decoder = MODELS.from_dict(classifier_cfg)
+        # self.decoder = PlainDecoder(num_classes=num_classes)
+        self.lane_classifier = MODELS.from_dict(lane_classifier_cfg)
+        # self.lane_classifier = RESALaneExist(num_output=num_classes - 1, flattened_size=flattened_size)
+
+    def forward(self, x):
+        x = self.backbone(x)
+        if isinstance(x, dict):
+            x = x['out']
+        x = self.channel_reducer(x)
+        x = self.spatial_conv(x)
+
+        res = {'out': self.decoder(x),
+               'lane': self.lane_classifier(x)}
+
+        return res

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/lane_detection/utils.py

@@ -0,0 +1,14 @@
+# Setup to temporarily avoid recursive imports
+
+
+def lane_pruning(existence, existence_conf, max_lane):
+    # Prune lanes based on confidence (a max number constrain for lanes in an image)
+    # Maybe too slow (but should be faster than topk/sort),
+    # consider batch size >> max number of lanes
+    while (existence.sum(dim=1) > max_lane).sum() > 0:
+        indices = (existence.sum(dim=1, keepdim=True) > max_lane).expand_as(existence) * \
+                  (existence_conf == existence_conf.min(dim=1, keepdim=True).values)
+        existence[indices] = 0
+        existence_conf[indices] = 1.1  # So we can keep using min
+
+    return existence, existence_conf

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/mlp.py

@@ -0,0 +1,22 @@
+from torch import nn
+import torch.nn.functional as F
+
+from .builder import MODELS
+
+
+@MODELS.register()
+class MLP(nn.Module):
+    # Modified from facebookresearch/detr
+    # Very simple multi-layer perceptron (also called FFN)
+
+    def __init__(self, input_dim, hidden_dim, output_dim, num_layers):
+        super().__init__()
+        self.num_layers = num_layers
+        h = [hidden_dim] * (num_layers - 1)
+        self.layers = nn.ModuleList(nn.Linear(n, k) for n, k in zip([input_dim] + h, h + [output_dim]))
+
+    def forward(self, x):
+        for i, layer in enumerate(self.layers):
+            x = F.relu(layer(x)) if i < self.num_layers - 1 else layer(x)
+
+        return x

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/mobilenet_v2.py

@@ -0,0 +1,156 @@
+# Modified from mmsegmentation code, referenced from torchvision
+
+import torch.nn as nn
+from .builder import MODELS
+from ._utils import make_divisible
+from .common_models import InvertedResidual
+from .utils import load_state_dict_from_url
+
+
+@MODELS.register()
+class MobileNetV2Encoder(nn.Module):
+    """MobileNetV2 backbone (up to second-to-last feature map).
+    This backbone is the implementation of
+    `MobileNetV2: Inverted Residuals and Linear Bottlenecks
+    <https://arxiv.org/abs/1801.04381>`_.
+    Args:
+        widen_factor (float): Width multiplier, multiply number of
+            channels in each layer by this amount. Default: 1.0.
+        strides (Sequence[int], optional): Strides of the first block of each
+            layer. If not specified, default config in ``arch_setting`` will
+            be used.
+        dilations (Sequence[int]): Dilation of each layer.
+        out_indices (None or Sequence[int]): Output from which stages.
+            Default: (7, ).
+        frozen_stages (int): Stages to be frozen (all param fixed).
+            Default: -1, which means not freezing any parameters.
+        norm_eval (bool): Whether to set norm layers to eval mode, namely,
+            freeze running stats (mean and var). Note: Effect on Batch Norm
+            and its variants only. Default: False.
+        pretrained (str, optional): model pretrained path. Default: None
+        out_stride (int): the output stride of the output feature map
+    """
+    # Parameters to build layers. 3 parameters are needed to construct a
+    # layer, from left to right: expand_ratio, channel, num_blocks.
+    arch_settings = [[1, 16, 1], [6, 24, 2], [6, 32, 3], [6, 64, 4],
+                     [6, 96, 3], [6, 160, 3], [6, 320, 1]]
+
+    def __init__(self, widen_factor=1., strides=(1, 2, 2, 2, 1, 2, 1), dilations=(1, 1, 1, 1, 1, 1, 1),
+                 out_indices=(1, 2, 4, 6), frozen_stages=-1, norm_eval=False, pretrained=None,
+                 progress=True, out_stride=0):
+        super(MobileNetV2Encoder, self).__init__()
+        self.pretrained = pretrained
+        self.widen_factor = widen_factor
+        self.strides = strides
+        self.dilations = dilations
+        assert len(strides) == len(dilations) == len(self.arch_settings)
+        self.out_indices = out_indices
+        for index in out_indices:
+            if index not in range(0, 7):
+                raise ValueError('the item in out_indices must in range(0, 7). But received {index}')
+        if frozen_stages not in range(-1, 7):
+            raise ValueError('frozen_stages must be in range(-1, 7). But received {frozen_stages}')
+        self.out_indices = out_indices
+        self.frozen_stages = frozen_stages
+        self.norm_eval = norm_eval
+        self.out_stride = out_stride
+        self.in_channels = make_divisible(32 * widen_factor, 8)
+        self.conv1 = nn.Sequential(
+            nn.Conv2d(in_channels=3, out_channels=self.in_channels, kernel_size=3, stride=2, padding=1, bias=False),
+            nn.BatchNorm2d(self.in_channels),
+            nn.ReLU6()
+        )
+        self.layers = []
+        for i, layer_cfg in enumerate(self.arch_settings):
+            expand_ratio, channel, num_blocks = layer_cfg
+            stride = self.strides[i]
+            dilation = self.dilations[i]
+            out_channels = make_divisible(channel * widen_factor, 8)
+            inverted_res_layer = self.make_layer(
+                out_channels=out_channels,
+                num_blocks=num_blocks,
+                stride=stride,
+                dilation=dilation,
+                expand_ratio=expand_ratio)
+            layer_name = f'layer{i + 1}'
+            self.add_module(layer_name, inverted_res_layer)
+            self.layers.append(layer_name)
+
+        if self.pretrained is None:
+            self.weight_initialization()
+        else:
+            self.load_pretrained(progress=progress)
+
+    def load_pretrained(self, progress):
+        state_dict = load_state_dict_from_url(self.pretrained, progress=progress)
+        self_state_dict = self.state_dict()
+        self_keys = list(self_state_dict.keys())
+        for i, (_, v) in enumerate(state_dict.items()):
+            if i > len(self_keys) - 1:
+                break
+            self_state_dict[self_keys[i]] = v
+        self.load_state_dict(self_state_dict)
+
+    def weight_initialization(self):
+        # weight initialization
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                nn.init.kaiming_normal_(m.weight, mode="fan_out")
+                if m.bias is not None:
+                    nn.init.zeros_(m.bias)
+            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):
+                nn.init.ones_(m.weight)
+                nn.init.zeros_(m.bias)
+            elif isinstance(m, nn.Linear):
+                nn.init.normal_(m.weight, 0, 0.01)
+                nn.init.zeros_(m.bias)
+
+    def make_layer(self, out_channels, num_blocks, stride, dilation,
+                   expand_ratio):
+        """Stack InvertedResidual blocks to build a layer for MobileNetV2.
+        Args:
+            out_channels (int): out_channels of block.
+            num_blocks (int): Number of blocks.
+            stride (int): Stride of the first block.
+            dilation (int): Dilation of the first block.
+            expand_ratio (int): Expand the number of channels of the
+                hidden layer in InvertedResidual by this ratio.
+        """
+        layers = []
+        for i in range(num_blocks):
+            layers.append(
+                InvertedResidual(
+                    self.in_channels,
+                    out_channels,
+                    stride if i == 0 else 1,
+                    expand_ratio=expand_ratio,
+                    dilation=dilation if i == 0 else 1)
+            )
+            self.in_channels = out_channels
+
+        return nn.Sequential(*layers)
+
+    def forward(self, x):
+        x = self.conv1(x)
+
+        outs = []
+        for i, layer_name in enumerate(self.layers):
+            layer = getattr(self, layer_name)
+            x = layer(x)
+            if i in self.out_indices:
+                outs.append(x)
+
+        if len(outs) == 1:
+            return outs[0]
+        else:
+            return tuple(outs)
+
+    def _freeze_stages(self):
+        if self.frozen_stages >= 0:
+            for param in self.conv1.parameters():
+                param.requires_grad = False
+        for i in range(1, self.frozen_stages + 1):
+            layer = getattr(self, f'layer{i}')
+            layer.eval()
+            for param in layer.parameters():
+                param.requires_grad = False

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/mobilenet_v3.py

@@ -0,0 +1,177 @@
+# Modified from mmsegmentation code, referenced from torchvision
+
+import torch
+import torch.nn as nn
+
+from .builder import MODELS
+from .common_models import InvertedResidualV3
+
+
+@MODELS.register()
+class MobileNetV3Encoder(nn.Module):
+    """MobileNetV3 backbone (keep the last 1x1)).
+    This backbone is the improved implementation of `Searching for MobileNetV3
+    <https://ieeexplore.ieee.org/document/9008835>`_.
+    Args:
+        arch (str): Architecture of mobilnetv3, from {'small', 'large'}.
+            Default: 'small'.
+        out_indices (tuple[int]): Output from which layer.
+            Default: (0, 1, 12).
+        frozen_stages (int): Stages to be frozen (all param fixed).
+            Default: -1, which means not freezing any parameters.
+        norm_eval (bool): Whether to set norm layers to eval mode, namely,
+            freeze running stats (mean and var). Note: Effect on Batch Norm
+            and its variants only. Default: False.
+        pretrained (str, optional): model pretrained path. Default: None
+    """
+    # MobileNet V3 for segmentation
+    # Parameters to build each block:
+    #     [kernel size, mid channels, out channels, with_se, act type, stride, dilated]
+    arch_settings = {
+        'small': [[3, 16, 16, True, 'ReLU'],  # block0 layer1 os=4
+                  [3, 72, 24, False, 'ReLU'],  # block1 layer2 os=8
+                  [3, 88, 24, False, 'ReLU'],
+                  [5, 96, 40, True, 'HSwish'],  # block2 layer4 os=16
+                  [5, 240, 40, True, 'HSwish'],
+                  [5, 240, 40, True, 'HSwish'],
+                  [5, 120, 48, True, 'HSwish'],  # block3 layer7 os=16
+                  [5, 144, 48, True, 'HSwish'],
+                  [5, 288, 96, True, 'HSwish'],  # block4 layer9 os=32
+                  [5, 576, 96, True, 'HSwish'],
+                  [5, 576, 96, True, 'HSwish']],
+        'large': [[3, 16, 16, False, 'ReLU'],  # block0 layer1 os=2
+                  [3, 64, 24, False, 'ReLU'],  # block1 layer2 os=4
+                  [3, 72, 24, False, 'ReLU'],
+                  [5, 72, 40, True, 'ReLU'],  # block2 layer4 os=8
+                  [5, 120, 40, True, 'ReLU'],
+                  [5, 120, 40, True, 'ReLU'],
+                  [3, 240, 80, False, 'HSwish'],  # block3 layer7 os=16
+                  [3, 200, 80, False, 'HSwish'],
+                  [3, 184, 80, False, 'HSwish'],
+                  [3, 184, 80, False, 'HSwish'],
+                  [3, 480, 112, True, 'HSwish'],  # block4 layer11 os=16
+                  [3, 672, 112, True, 'HSwish'],
+                  [5, 672, 160, True, 'HSwish'],  # block5 layer13 os=32
+                  [5, 960, 160, True, 'HSwish'],
+                  [5, 960, 160, True, 'HSwish']]
+    }
+
+    def __init__(self, arch='small', out_indices=(12, ), frozen_stages=-1, reduction_factor=1,
+                 norm_eval=False, pretrained=None,
+                 strides=(2, 2, 1, 2, 1, 1, 1, 1, 2, 1, 1),
+                 dilations=(1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2)):
+        super(MobileNetV3Encoder, self).__init__()
+
+        self.pretrained = pretrained
+        assert arch in self.arch_settings
+        assert isinstance(reduction_factor, int) and reduction_factor > 0
+        self.strides = strides
+        self.dilations = dilations
+
+        for index in out_indices:
+            if index not in range(0, len(self.arch_settings[arch]) + 2):
+                raise ValueError(
+                    'the item in out_indices must in '
+                    f'range(0, {len(self.arch_settings[arch]) + 2}). '
+                    f'But received {index}')
+
+        if frozen_stages not in range(-1, len(self.arch_settings[arch]) + 2):
+            raise ValueError('frozen_stages must be in range(-1, '
+                             f'{len(self.arch_settings[arch]) + 2}). '
+                             f'But received {frozen_stages}')
+        self.arch = arch
+        self.out_indices = out_indices
+        self.frozen_stages = frozen_stages
+        self.reduction_factor = reduction_factor
+        self.norm_eval = norm_eval
+        self.layers = self._make_layer()
+
+        if self.pretrained is None:
+            self.weight_initialization()
+        else:
+            self.load_pretrained()
+
+    def _make_layer(self):
+        layers = []
+
+        # build the first layer (layer0)
+        in_channels = 16
+        layer = nn.Sequential(
+            nn.Conv2d(in_channels=3, out_channels=in_channels, kernel_size=3, stride=2, padding=1, bias=False),
+            nn.BatchNorm2d(in_channels),
+            nn.Hardswish()
+        )
+        self.add_module('layer0', layer)
+        layers.append('layer0')
+
+        layer_setting = self.arch_settings[self.arch]
+        for i, params in enumerate(layer_setting):
+            (kernel_size, mid_channels, out_channels, with_se, act) = params
+            stride = self.strides[i]
+            dilation = self.dilations[i]
+
+            if self.arch == 'large' and i >= 12 or self.arch == 'small' and i >= 8:
+                mid_channels = mid_channels // self.reduction_factor
+                out_channels = out_channels // self.reduction_factor
+            layer = InvertedResidualV3(in_channels=in_channels, out_channels=out_channels, mid_channels=mid_channels,
+                                       kernel_size=kernel_size, stride=stride, with_se=with_se, act=act,
+                                       with_expand_conv=(in_channels != mid_channels), dilation=dilation)
+            in_channels = out_channels
+            layer_name = 'layer{}'.format(i + 1)
+            self.add_module(layer_name, layer)
+            layers.append(layer_name)
+
+        # build the last layer
+        # block5 layer12 os=32 for small model
+        # block6 layer16 os=32 for large model
+        out_channels = 576 if self.arch == 'small' else 960
+        layer = nn.Sequential(
+            nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=1, stride=1,
+                      padding=0, bias=False),
+            nn.BatchNorm2d(out_channels),
+            nn.Hardswish()
+        )
+        layer_name = 'layer{}'.format(len(layer_setting) + 1)
+        self.add_module(layer_name, layer)
+        layers.append(layer_name)
+
+        return layers
+
+    def forward(self, x):
+        outs = []
+        for i, layer_name in enumerate(self.layers):
+            layer = getattr(self, layer_name)
+            x = layer(x)
+            if i in self.out_indices:
+                outs.append(x)
+        return outs[-1]
+
+    def _freeze_stages(self):
+        for i in range(self.frozen_stages + 1):
+            layer = getattr(self, f'layer{i}')
+            layer.eval()
+            for param in layer.parameters():
+                param.requires_grad = False
+
+    def load_pretrained(self):
+        state_dict = torch.load(self.pretrained)
+        self_state_dict = self.state_dict()
+        self_keys = list(self_state_dict.keys())
+        for i, (_, v) in enumerate(state_dict.items()):
+            if i > len(self_keys) - 1:
+                break
+            self_state_dict[self_keys[i]] = v
+        self.load_state_dict(self_state_dict)
+
+    def weight_initialization(self):
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                nn.init.kaiming_normal_(m.weight, mode="fan_out")
+                if m.bias is not None:
+                    nn.init.zeros_(m.bias)
+            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):
+                nn.init.ones_(m.weight)
+                nn.init.zeros_(m.bias)
+            elif isinstance(m, nn.Linear):
+                nn.init.normal_(m.weight, 0, 0.01)
+                nn.init.zeros_(m.bias)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/rep_vgg.py

@@ -0,0 +1,340 @@
+# https://github.com/DingXiaoH/RepVGG
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import numpy as np
+
+from .builder import MODELS
+
+
+def conv_bn(in_channels, out_channels, kernel_size, stride, padding, groups=1):
+    result = nn.Sequential()
+    result.add_module('conv', nn.Conv2d(in_channels=in_channels, out_channels=out_channels,
+                                        kernel_size=kernel_size, stride=stride, padding=padding, groups=groups,
+                                        bias=False))
+    result.add_module('bn', nn.BatchNorm2d(num_features=out_channels))
+    return result
+
+
+class SEBlock(nn.Module):
+    # https://openaccess.thecvf.com/content_cvpr_2018/html/Hu_Squeeze-and-Excitation_Networks_CVPR_2018_paper.html
+    def __init__(self, input_channels, internal_neurons):
+        super(SEBlock, self).__init__()
+        self.down = nn.Conv2d(in_channels=input_channels, out_channels=internal_neurons, kernel_size=1, stride=1,
+                              bias=True)
+        self.up = nn.Conv2d(in_channels=internal_neurons, out_channels=input_channels, kernel_size=1, stride=1,
+                            bias=True)
+        self.input_channels = input_channels
+
+    def forward(self, inputs):
+        x = F.avg_pool2d(inputs, kernel_size=inputs.size(3))
+        x = self.down(x)
+        x = F.relu(x)
+        x = self.up(x)
+        x = torch.sigmoid(x)
+        x = x.view(-1, self.input_channels, 1, 1)
+        return inputs * x
+
+
+class RepVGGBlock(nn.Module):
+
+    def __init__(self, in_channels, out_channels, kernel_size,
+                 stride=1, padding=0, dilation=1, groups=1, padding_mode='zeros', deploy=False, use_se=False):
+        super(RepVGGBlock, self).__init__()
+        self.deploy = deploy
+        self.groups = groups
+        self.in_channels = in_channels
+
+        assert kernel_size == 3
+        assert padding == 1
+
+        padding_11 = padding - kernel_size // 2
+
+        self.nonlinearity = nn.ReLU()
+
+        if use_se:
+            self.se = SEBlock(out_channels, internal_neurons=out_channels // 16)
+        else:
+            self.se = nn.Identity()
+
+        if deploy:
+            self.rbr_reparam = nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size,
+                                         stride=stride,
+                                         padding=padding, dilation=dilation, groups=groups, bias=True,
+                                         padding_mode=padding_mode)
+
+        else:
+            self.rbr_identity = nn.BatchNorm2d(
+                num_features=in_channels) if out_channels == in_channels and stride == 1 else None
+            self.rbr_dense = conv_bn(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size,
+                                     stride=stride, padding=padding, groups=groups)
+            self.rbr_1x1 = conv_bn(in_channels=in_channels, out_channels=out_channels, kernel_size=1, stride=stride,
+                                   padding=padding_11, groups=groups)
+            print('RepVGG Block, identity = ', self.rbr_identity)
+
+    def forward(self, inputs):
+        if hasattr(self, 'rbr_reparam'):
+            return self.nonlinearity(self.se(self.rbr_reparam(inputs)))
+
+        if self.rbr_identity is None:
+            id_out = 0
+        else:
+            id_out = self.rbr_identity(inputs)
+
+        return self.nonlinearity(self.se(self.rbr_dense(inputs) + self.rbr_1x1(inputs) + id_out))
+
+    #   Optional. This improves the accuracy and facilitates quantization.
+    #   1.  Cancel the original weight decay on rbr_dense.conv.weight and rbr_1x1.conv.weight.
+    #   2.  Use like this.
+    #       loss = criterion(....)
+    #       for every RepVGGBlock blk:
+    #           loss += weight_decay_coefficient * 0.5 * blk.get_cust_L2()
+    #       optimizer.zero_grad()
+    #       loss.backward()
+    def get_custom_L2(self):
+        K3 = self.rbr_dense.conv.weight
+        K1 = self.rbr_1x1.conv.weight
+        t3 = (self.rbr_dense.bn.weight / ((self.rbr_dense.bn.running_var + self.rbr_dense.bn.eps).sqrt())).reshape(-1,
+                                                                                                                   1, 1,
+                                                                                                                   1).detach()
+        t1 = (self.rbr_1x1.bn.weight / ((self.rbr_1x1.bn.running_var + self.rbr_1x1.bn.eps).sqrt())).reshape(-1, 1, 1,
+                                                                                                             1).detach()
+        # The L2 loss of the "circle" of weights in 3x3 kernel. Use regular L2 on them.
+        l2_loss_circle = (K3 ** 2).sum() - (K3[:, :, 1:2, 1:2] ** 2).sum()
+        # The equivalent resultant central point of 3x3 kernel.
+        eq_kernel = K3[:, :, 1:2, 1:2] * t3 + K1 * t1
+        # Normalize for an L2 coefficient comparable to regular L2.
+        l2_loss_eq_kernel = (eq_kernel ** 2 / (t3 ** 2 + t1 ** 2)).sum()
+        return l2_loss_eq_kernel + l2_loss_circle
+
+    #   This func derives the equivalent kernel and bias in a DIFFERENTIABLE way.
+    #   You can get the equivalent kernel and bias at any time and do whatever you want,
+    #   for example, apply some penalties or constraints during training, just like you do to the other models.
+    #   May be useful for quantization or pruning.
+    def get_equivalent_kernel_bias(self):
+        kernel3x3, bias3x3 = self._fuse_bn_tensor(self.rbr_dense)
+        kernel1x1, bias1x1 = self._fuse_bn_tensor(self.rbr_1x1)
+        kernelid, biasid = self._fuse_bn_tensor(self.rbr_identity)
+        return kernel3x3 + self._pad_1x1_to_3x3_tensor(kernel1x1) + kernelid, bias3x3 + bias1x1 + biasid
+
+    def _pad_1x1_to_3x3_tensor(self, kernel1x1):
+        if kernel1x1 is None:
+            return 0
+        else:
+            return torch.nn.functional.pad(kernel1x1, [1, 1, 1, 1])
+
+    def _fuse_bn_tensor(self, branch):
+        if branch is None:
+            return 0, 0
+        if isinstance(branch, nn.Sequential):
+            kernel = branch.conv.weight
+            running_mean = branch.bn.running_mean
+            running_var = branch.bn.running_var
+            gamma = branch.bn.weight
+            beta = branch.bn.bias
+            eps = branch.bn.eps
+        else:
+            assert isinstance(branch, nn.BatchNorm2d)
+            if not hasattr(self, 'id_tensor'):
+                input_dim = self.in_channels // self.groups
+                kernel_value = np.zeros((self.in_channels, input_dim, 3, 3), dtype=np.float32)
+                for i in range(self.in_channels):
+                    kernel_value[i, i % input_dim, 1, 1] = 1
+                self.id_tensor = torch.from_numpy(kernel_value).to(branch.weight.device)
+            kernel = self.id_tensor
+            running_mean = branch.running_mean
+            running_var = branch.running_var
+            gamma = branch.weight
+            beta = branch.bias
+            eps = branch.eps
+        std = (running_var + eps).sqrt()
+        t = (gamma / std).reshape(-1, 1, 1, 1)
+        return kernel * t, beta - running_mean * gamma / std
+
+    def switch_to_deploy(self):
+        if hasattr(self, 'rbr_reparam'):
+            return
+        kernel, bias = self.get_equivalent_kernel_bias()
+        self.rbr_reparam = nn.Conv2d(in_channels=self.rbr_dense.conv.in_channels,
+                                     out_channels=self.rbr_dense.conv.out_channels,
+                                     kernel_size=self.rbr_dense.conv.kernel_size, stride=self.rbr_dense.conv.stride,
+                                     padding=self.rbr_dense.conv.padding, dilation=self.rbr_dense.conv.dilation,
+                                     groups=self.rbr_dense.conv.groups, bias=True)
+        self.rbr_reparam.weight.data = kernel
+        self.rbr_reparam.bias.data = bias
+        for para in self.parameters():
+            para.detach_()
+        self.__delattr__('rbr_dense')
+        self.__delattr__('rbr_1x1')
+        if hasattr(self, 'rbr_identity'):
+            self.__delattr__('rbr_identity')
+        if hasattr(self, 'id_tensor'):
+            self.__delattr__('id_tensor')
+        self.deploy = True
+
+
+class RepVGG(nn.Module):
+    def __init__(self, num_blocks, num_classes=1000, width_multiplier=None, override_groups_map=None, deploy=False,
+                 use_se=False):
+        super(RepVGG, self).__init__()
+
+        assert len(width_multiplier) == 4
+
+        self.deploy = deploy
+        self.override_groups_map = override_groups_map or dict()
+        self.use_se = use_se
+
+        assert 0 not in self.override_groups_map
+
+        self.in_planes = min(64, int(64 * width_multiplier[0]))
+
+        self.stage0 = RepVGGBlock(in_channels=3, out_channels=self.in_planes, kernel_size=3, stride=2, padding=1,
+                                  deploy=self.deploy, use_se=self.use_se)
+        self.cur_layer_idx = 1
+        self.stage1 = self._make_stage(int(64 * width_multiplier[0]), num_blocks[0], stride=2)
+        self.stage2 = self._make_stage(int(128 * width_multiplier[1]), num_blocks[1], stride=2)
+        self.stage3 = self._make_stage(int(256 * width_multiplier[2]), num_blocks[2], stride=2)
+        self.stage4 = self._make_stage(int(512 * width_multiplier[3]), num_blocks[3], stride=2)
+        self.gap = nn.AdaptiveAvgPool2d(output_size=1)
+        self.linear = nn.Linear(int(512 * width_multiplier[3]), num_classes)
+
+    def _make_stage(self, planes, num_blocks, stride):
+        strides = [stride] + [1] * (num_blocks - 1)
+        blocks = []
+        for stride in strides:
+            cur_groups = self.override_groups_map.get(self.cur_layer_idx, 1)
+            blocks.append(RepVGGBlock(in_channels=self.in_planes, out_channels=planes, kernel_size=3,
+                                      stride=stride, padding=1, groups=cur_groups, deploy=self.deploy,
+                                      use_se=self.use_se))
+            self.in_planes = planes
+            self.cur_layer_idx += 1
+        return nn.Sequential(*blocks)
+
+    def forward(self, x):
+        out = self.stage0(x)
+        out = self.stage1(out)
+        out = self.stage2(out)
+        out = self.stage3(out)
+        out = self.stage4(out)
+        out = self.gap(out)
+        out = out.view(out.size(0), -1)
+        out = self.linear(out)
+        return out
+
+
+optional_groupwise_layers = [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26]
+g2_map = {l: 2 for l in optional_groupwise_layers}
+g4_map = {l: 4 for l in optional_groupwise_layers}
+
+
+def create_RepVGG_A0(deploy=False):
+    return RepVGG(num_blocks=[2, 4, 14, 1], num_classes=1000,
+                  width_multiplier=[0.75, 0.75, 0.75, 2.5], override_groups_map=None, deploy=deploy)
+
+
+def create_RepVGG_A1(deploy=False):
+    return RepVGG(num_blocks=[2, 4, 14, 1], num_classes=1000,
+                  width_multiplier=[1, 1, 1, 2.5], override_groups_map=None, deploy=deploy)
+
+
+def create_RepVGG_A2(deploy=False):
+    return RepVGG(num_blocks=[2, 4, 14, 1], num_classes=1000,
+                  width_multiplier=[1.5, 1.5, 1.5, 2.75], override_groups_map=None, deploy=deploy)
+
+
+def create_RepVGG_B0(deploy=False):
+    return RepVGG(num_blocks=[4, 6, 16, 1], num_classes=1000,
+                  width_multiplier=[1, 1, 1, 2.5], override_groups_map=None, deploy=deploy)
+
+
+def create_RepVGG_B1(deploy=False):
+    return RepVGG(num_blocks=[4, 6, 16, 1], num_classes=1000,
+                  width_multiplier=[2, 2, 2, 4], override_groups_map=None, deploy=deploy)
+
+
+def create_RepVGG_B1g2(deploy=False):
+    return RepVGG(num_blocks=[4, 6, 16, 1], num_classes=1000,
+                  width_multiplier=[2, 2, 2, 4], override_groups_map=g2_map, deploy=deploy)
+
+
+def create_RepVGG_B2(deploy=False):
+    return RepVGG(num_blocks=[4, 6, 16, 1], num_classes=1000,
+                  width_multiplier=[2.5, 2.5, 2.5, 5], override_groups_map=None, deploy=deploy)
+
+
+func_dict = {
+    'RepVGG-A0': create_RepVGG_A0,
+    'RepVGG-A1': create_RepVGG_A1,
+    'RepVGG-A2': create_RepVGG_A2,
+    'RepVGG-B0': create_RepVGG_B0,
+    'RepVGG-B1': create_RepVGG_B1,
+    'RepVGG-B1g2': create_RepVGG_B1g2,
+    'RepVGG-B2': create_RepVGG_B2
+}
+
+pretrained_model_dict = {
+    'RepVGG-A0': 'RepVGG-A0-train.pth',
+    'RepVGG-A1': 'RepVGG-A1-train.pth',
+    'RepVGG-A2': 'RepVGG-A2-train.pth',
+    'RepVGG-B0': 'RepVGG-B0-train.pth',
+    'RepVGG-B1': 'RepVGG-B1-train.pth',
+    'RepVGG-B1g2': 'RepVGG-B1g2-train.pth',
+    'RepVGG-B2': 'RepVGG-B2-train.pth'
+}
+
+
+def get_RepVGG_func_by_name(name):
+    return func_dict[name]
+
+
+@MODELS.register()
+class RepVggEncoder(nn.Module):
+    def __init__(self, backbone_name, pretrained=False, deploy=False):
+        super(RepVggEncoder, self).__init__()
+        repvgg_fn = get_RepVGG_func_by_name(backbone_name)
+        self.encoder = repvgg_fn(deploy)
+        if pretrained:
+            checkpoint = torch.load(pretrained_model_dict[backbone_name])
+            if 'state_dict' in checkpoint:
+                checkpoint = checkpoint['state_dict']
+            ckpt = {k.replace('module.', ''): v for k, v in checkpoint.items()}  # strip the names
+            self.encoder.load_state_dict(ckpt)
+        # TODO: Refactor with feature extractor
+        self.layer0 = self.encoder.stage0
+        self.layer1 = self.encoder.stage1
+        self.layer2 = self.encoder.stage2
+        self.layer3 = self.encoder.stage3
+        self.layer4 = self.encoder.stage4
+
+        #   The last two stages should have stride=1 for semantic segmentation
+        #   Note that the stride of 1x1 should be the same as the 3x3
+        #   Use dilation following the implementation of PSPNet
+        secondlast_channel = 0
+        for n, m in self.layer3.named_modules():
+            if ('rbr_dense' in n or 'rbr_reparam' in n) and isinstance(m, nn.Conv2d):
+                m.dilation, m.padding, m.stride = (2, 2), (2, 2), (1, 1)
+                print('change dilation, padding, stride of ', n)
+                secondlast_channel = m.out_channels
+            elif 'rbr_1x1' in n and isinstance(m, nn.Conv2d):
+                m.stride = (1, 1)
+                print('change stride of ', n)
+        last_channel = 0
+        for n, m in self.layer4.named_modules():
+            if ('rbr_dense' in n or 'rbr_reparam' in n) and isinstance(m, nn.Conv2d):
+                m.dilation, m.padding, m.stride = (4, 4), (4, 4), (1, 1)
+                print('change dilation, padding, stride of ', n)
+                last_channel = m.out_channels
+            elif 'rbr_1x1' in n and isinstance(m, nn.Conv2d):
+                m.stride = (1, 1)
+                print('change stride of ', n)
+        self.fea_dim = last_channel
+
+    def forward(self, x):
+        x0 = self.layer0(x)
+        x1 = self.layer1(x0)
+        x2 = self.layer2(x1)
+        x3 = self.layer3(x2)
+        x4 = self.layer4(x3)
+
+        return x4

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/resnet.py

@@ -0,0 +1,427 @@
+import torch
+import torch.nn as nn
+from torch import Tensor
+from typing import List, Optional
+from .utils import load_state_dict_from_url
+try:
+    from ..common import warnings
+except ImportError:
+    import warnings
+
+
+__all__ = ['ResNet', 'resnet18', 'resnet18_reduced', 'resnet34', 'resnet50', 'resnet101',
+           'resnet152', 'resnext50_32x4d', 'resnext101_32x8d',
+           'wide_resnet50_2', 'wide_resnet101_2']
+
+
+model_urls = {
+    'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth',
+    'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth',
+    'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',
+    'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',
+    'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',
+    'resnext50_32x4d': 'https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth',
+    'resnext101_32x8d': 'https://download.pytorch.org/models/resnext101_32x8d-8ba56ff5.pth',
+    'wide_resnet50_2': 'https://download.pytorch.org/models/wide_resnet50_2-95faca4d.pth',
+    'wide_resnet101_2': 'https://download.pytorch.org/models/wide_resnet101_2-32ee1156.pth',
+}
+
+
+class FrozenBatchNorm2d(torch.nn.Module):
+    """
+    BatchNorm2d where the batch statistics and the affine parameters
+    are fixed
+    """
+
+    def __init__(
+        self,
+        num_features: int,
+        eps: float = 1e-5,
+        n: Optional[int] = None,
+    ):
+        # n=None for backward-compatibility
+        if n is not None:
+            warnings.warn("`n` argument is deprecated and has been renamed `num_features`",
+                          DeprecationWarning)
+            num_features = n
+        super(FrozenBatchNorm2d, self).__init__()
+        self.eps = eps
+        self.register_buffer("weight", torch.ones(num_features))
+        self.register_buffer("bias", torch.zeros(num_features))
+        self.register_buffer("running_mean", torch.zeros(num_features))
+        self.register_buffer("running_var", torch.ones(num_features))
+
+    def _load_from_state_dict(
+        self,
+        state_dict: dict,
+        prefix: str,
+        local_metadata: dict,
+        strict: bool,
+        missing_keys: List[str],
+        unexpected_keys: List[str],
+        error_msgs: List[str],
+    ):
+        num_batches_tracked_key = prefix + 'num_batches_tracked'
+        if num_batches_tracked_key in state_dict:
+            del state_dict[num_batches_tracked_key]
+
+        super(FrozenBatchNorm2d, self)._load_from_state_dict(
+            state_dict, prefix, local_metadata, strict,
+            missing_keys, unexpected_keys, error_msgs)
+
+    def forward(self, x: Tensor) -> Tensor:
+        # move reshapes to the beginning
+        # to make it fuser-friendly
+        w = self.weight.reshape(1, -1, 1, 1)
+        b = self.bias.reshape(1, -1, 1, 1)
+        rv = self.running_var.reshape(1, -1, 1, 1)
+        rm = self.running_mean.reshape(1, -1, 1, 1)
+        scale = w * (rv + self.eps).rsqrt()
+        bias = b - rm * scale
+        return x * scale + bias
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}({self.weight.shape[0]}, eps={self.eps})"
+
+
+def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1):
+    """3x3 convolution with padding"""
+    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
+                     padding=dilation, groups=groups, bias=False, dilation=dilation)
+
+
+def conv1x1(in_planes, out_planes, stride=1):
+    """1x1 convolution"""
+    return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)
+
+
+class BasicBlock(nn.Module):
+    expansion = 1
+
+    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,
+                 base_width=64, dilation=1, norm_layer=None):
+        super(BasicBlock, self).__init__()
+        if norm_layer is None:
+            norm_layer = nn.BatchNorm2d
+        if groups != 1 or base_width != 64:
+            raise ValueError('BasicBlock only supports groups=1 and base_width=64')
+        # if dilation > 1:
+        #     raise NotImplementedError("Dilation > 1 not supported in BasicBlock")
+        # Both self.conv1 and self.downsample layers downsample the input when stride != 1
+        self.conv1 = conv3x3(inplanes, planes, stride, 1, dilation)
+        self.bn1 = norm_layer(planes)
+        self.relu = nn.ReLU(inplace=True)
+        self.conv2 = conv3x3(planes, planes)
+        self.bn2 = norm_layer(planes)
+        self.downsample = downsample
+        self.stride = stride
+
+    def forward(self, x):
+        identity = x
+
+        out = self.conv1(x)
+        out = self.bn1(out)
+        out = self.relu(out)
+
+        out = self.conv2(out)
+        out = self.bn2(out)
+
+        if self.downsample is not None:
+            identity = self.downsample(x)
+
+        out += identity
+        out = self.relu(out)
+
+        return out
+
+
+class Bottleneck(nn.Module):
+    # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2)
+    # while original implementation places the stride at the first 1x1 convolution(self.conv1)
+    # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385.
+    # This variant is also known as ResNet V1.5 and improves accuracy according to
+    # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch.
+
+    expansion = 4
+
+    def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1,
+                 base_width=64, dilation=1, norm_layer=None):
+        super(Bottleneck, self).__init__()
+        if norm_layer is None:
+            norm_layer = nn.BatchNorm2d
+        width = int(planes * (base_width / 64.)) * groups
+        # Both self.conv2 and self.downsample layers downsample the input when stride != 1
+        self.conv1 = conv1x1(inplanes, width)
+        self.bn1 = norm_layer(width)
+        self.conv2 = conv3x3(width, width, stride, groups, dilation)
+        self.bn2 = norm_layer(width)
+        self.conv3 = conv1x1(width, planes * self.expansion)
+        self.bn3 = norm_layer(planes * self.expansion)
+        self.relu = nn.ReLU(inplace=True)
+        self.downsample = downsample
+        self.stride = stride
+
+    def forward(self, x):
+        identity = x
+
+        out = self.conv1(x)
+        out = self.bn1(out)
+        out = self.relu(out)
+
+        out = self.conv2(out)
+        out = self.bn2(out)
+        out = self.relu(out)
+
+        out = self.conv3(out)
+        out = self.bn3(out)
+
+        if self.downsample is not None:
+            identity = self.downsample(x)
+
+        out += identity
+        out = self.relu(out)
+
+        return out
+
+
+class ResNet(nn.Module):
+    def __init__(self, block, layers, num_classes=1000, zero_init_residual=False,
+                 groups=1, width_per_group=64, replace_stride_with_dilation=None,
+                 norm_layer=None, channels=None):
+        super(ResNet, self).__init__()
+        if channels is None:  # To keep BC
+            channels = [64, 128, 256, 512]
+        if norm_layer is None:
+            norm_layer = nn.BatchNorm2d
+        self._norm_layer = norm_layer
+
+        self.inplanes = channels[0]
+        self.dilation = 1
+        if replace_stride_with_dilation is None:
+            # each element in the tuple indicates if we should replace
+            # the 2x2 stride with a dilated convolution instead
+            replace_stride_with_dilation = [False, False, False]
+        if len(replace_stride_with_dilation) != 3:
+            raise ValueError("replace_stride_with_dilation should be None "
+                             "or a 3-element tuple, got {}".format(replace_stride_with_dilation))
+        self.groups = groups
+        self.base_width = width_per_group
+        self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3,
+                               bias=False)
+        self.bn1 = norm_layer(self.inplanes)
+        self.relu = nn.ReLU(inplace=True)
+        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
+        self.layer1 = self._make_layer(block, channels[0], layers[0])
+        self.layer2 = self._make_layer(block, channels[1], layers[1], stride=2,
+                                       dilate=replace_stride_with_dilation[0])
+        self.layer3 = self._make_layer(block, channels[2], layers[2], stride=2,
+                                       dilate=replace_stride_with_dilation[1])
+        self.layer4 = self._make_layer(block, channels[3], layers[3], stride=2,
+                                       dilate=replace_stride_with_dilation[2])
+        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
+        self.fc = nn.Linear(channels[3] * block.expansion, num_classes)
+
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
+            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):
+                nn.init.constant_(m.weight, 1)
+                nn.init.constant_(m.bias, 0)
+
+        # Zero-initialize the last BN in each residual branch,
+        # so that the residual branch starts with zeros, and each residual block behaves like an identity.
+        # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677
+        if zero_init_residual:
+            for m in self.modules():
+                if isinstance(m, Bottleneck):
+                    nn.init.constant_(m.bn3.weight, 0)
+                elif isinstance(m, BasicBlock):
+                    nn.init.constant_(m.bn2.weight, 0)
+
+    def _make_layer(self, block, planes, blocks, stride=1, dilate=False):
+        norm_layer = self._norm_layer
+        downsample = None
+        previous_dilation = self.dilation
+        if dilate:
+            self.dilation *= stride
+            stride = 1
+        if stride != 1 or self.inplanes != planes * block.expansion:
+            downsample = nn.Sequential(
+                conv1x1(self.inplanes, planes * block.expansion, stride),
+                norm_layer(planes * block.expansion),
+            )
+
+        layers = []
+        layers.append(block(self.inplanes, planes, stride, downsample, self.groups,
+                            self.base_width, previous_dilation, norm_layer))
+        self.inplanes = planes * block.expansion
+        for _ in range(1, blocks):
+            layers.append(block(self.inplanes, planes, groups=self.groups,
+                                base_width=self.base_width, dilation=self.dilation,
+                                norm_layer=norm_layer))
+
+        return nn.Sequential(*layers)
+
+    def forward(self, x):
+        x = self.conv1(x)
+        x = self.bn1(x)
+        x = self.relu(x)
+        x = self.maxpool(x)
+
+        x = self.layer1(x)
+        x = self.layer2(x)
+        x = self.layer3(x)
+        x = self.layer4(x)
+
+        x = self.avgpool(x)
+        x = torch.flatten(x, 1)
+        x = self.fc(x)
+
+        return x
+
+
+def _resnet(arch, block, layers, pretrained, progress, **kwargs):
+    model = ResNet(block, layers, **kwargs)
+    if pretrained:
+        # TODO: load for reduced ResNets
+        state_dict = load_state_dict_from_url(model_urls[arch],
+                                              progress=progress)
+        model.load_state_dict(state_dict)
+        print('Loaded torchvision ImageNet pre-trained weights V1.')
+    return model
+
+
+def resnet18(pretrained=False, progress=True, **kwargs):
+    r"""ResNet-18 model from
+    `"Deep Residual Learning for Image Recognition" <https://arxiv.org/pdf/1512.03385.pdf>`_
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+    """
+    return _resnet('resnet18', BasicBlock, [2, 2, 2, 2], pretrained, progress,
+                   **kwargs)
+
+
+def resnet18_reduced(pretrained=False, progress=True, expansion=1, **kwargs):
+    r"""Reduced ResNet18 (LSTR modification, a 4x expansion yields some kind of ResNet-17)
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+        expansion (int): Expansion rate for reduced ResNet18
+    """
+    channels = [channel * expansion for channel in [16, 32, 64, 128]]
+    return _resnet('resnet18', BasicBlock, [1, 2, 2, 2], pretrained, progress, channels=channels, **kwargs)
+
+
+def resnet34(pretrained=False, progress=True, **kwargs):
+    r"""ResNet-34 model from
+    `"Deep Residual Learning for Image Recognition" <https://arxiv.org/pdf/1512.03385.pdf>`_
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+    """
+    return _resnet('resnet34', BasicBlock, [3, 4, 6, 3], pretrained, progress,
+                   **kwargs)
+
+
+def resnet50(pretrained=False, progress=True, **kwargs):
+    r"""ResNet-50 model from
+    `"Deep Residual Learning for Image Recognition" <https://arxiv.org/pdf/1512.03385.pdf>`_
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+    """
+    return _resnet('resnet50', Bottleneck, [3, 4, 6, 3], pretrained, progress,
+                   **kwargs)
+
+
+def resnet101(pretrained=False, progress=True, **kwargs):
+    r"""ResNet-101 model from
+    `"Deep Residual Learning for Image Recognition" <https://arxiv.org/pdf/1512.03385.pdf>`_
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+    """
+    return _resnet('resnet101', Bottleneck, [3, 4, 23, 3], pretrained, progress,
+                   **kwargs)
+
+
+def resnet152(pretrained=False, progress=True, **kwargs):
+    r"""ResNet-152 model from
+    `"Deep Residual Learning for Image Recognition" <https://arxiv.org/pdf/1512.03385.pdf>`_
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+    """
+    return _resnet('resnet152', Bottleneck, [3, 8, 36, 3], pretrained, progress,
+                   **kwargs)
+
+
+def resnext50_32x4d(pretrained=False, progress=True, **kwargs):
+    r"""ResNeXt-50 32x4d model from
+    `"Aggregated Residual Transformation for Deep Neural Networks" <https://arxiv.org/pdf/1611.05431.pdf>`_
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+    """
+    kwargs['groups'] = 32
+    kwargs['width_per_group'] = 4
+    return _resnet('resnext50_32x4d', Bottleneck, [3, 4, 6, 3],
+                   pretrained, progress, **kwargs)
+
+
+def resnext101_32x8d(pretrained=False, progress=True, **kwargs):
+    r"""ResNeXt-101 32x8d model from
+    `"Aggregated Residual Transformation for Deep Neural Networks" <https://arxiv.org/pdf/1611.05431.pdf>`_
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+    """
+    kwargs['groups'] = 32
+    kwargs['width_per_group'] = 8
+    return _resnet('resnext101_32x8d', Bottleneck, [3, 4, 23, 3],
+                   pretrained, progress, **kwargs)
+
+
+def wide_resnet50_2(pretrained=False, progress=True, **kwargs):
+    r"""Wide ResNet-50-2 model from
+    `"Wide Residual Networks" <https://arxiv.org/pdf/1605.07146.pdf>`_
+
+    The model is the same as ResNet except for the bottleneck number of channels
+    which is twice larger in every block. The number of channels in outer 1x1
+    convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048
+    channels, and in Wide ResNet-50-2 has 2048-1024-2048.
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+    """
+    kwargs['width_per_group'] = 64 * 2
+    return _resnet('wide_resnet50_2', Bottleneck, [3, 4, 6, 3],
+                   pretrained, progress, **kwargs)
+
+
+def wide_resnet101_2(pretrained=False, progress=True, **kwargs):
+    r"""Wide ResNet-101-2 model from
+    `"Wide Residual Networks" <https://arxiv.org/pdf/1605.07146.pdf>`_
+
+    The model is the same as ResNet except for the bottleneck number of channels
+    which is twice larger in every block. The number of channels in outer 1x1
+    convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048
+    channels, and in Wide ResNet-50-2 has 2048-1024-2048.
+
+    Args:
+        pretrained (bool): If True, returns a model pre-trained on ImageNet
+        progress (bool): If True, displays a progress bar of the download to stderr
+    """
+    kwargs['width_per_group'] = 64 * 2
+    return _resnet('wide_resnet101_2', Bottleneck, [3, 4, 23, 3],
+                   pretrained, progress, **kwargs)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/segmentation/__init__.py

@@ -0,0 +1,7 @@
+from .segmentation import *
+from .fcn import *
+from .deeplab import *
+from .deeplab_vgg import DeepLabV1, DeepLabV1Lane, SegRepVGG
+from .erfnet import ERFNet
+from .enet import ENet
+from .simple_seg_head import SimpleSegHead

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/segmentation/_utils.py

@@ -0,0 +1,70 @@
+import torch.nn as nn
+from torch import load
+from collections import OrderedDict
+
+
+class _EncoderDecoderModel(nn.Module):
+    def __init__(self):
+        super().__init__()
+
+    def _load_encoder(self, pretrained_weights):
+        if pretrained_weights is not None:  # Load pre-trained weights
+            try:
+                saved_weights = load(pretrained_weights)['model']
+            except FileNotFoundError:
+                raise FileNotFoundError('pretrained_weights is not there! '
+                                        'Please set pretrained_weights=None if you are only testing.')
+            original_weights = self.state_dict()
+            for key in saved_weights.keys():
+                if key in original_weights.keys():
+                    original_weights[key] = saved_weights[key]
+            self.load_state_dict(original_weights)
+        else:
+            print('No pre-training.')
+
+    def forward(self, x):
+        pass
+
+
+class _SimpleSegmentationModel(nn.Module):
+    def __init__(self, backbone, classifier, aux_classifier=None, recon_head=None,
+                 lane_classifier=None, channel_reducer=None, scnn_layer=None):
+        super(_SimpleSegmentationModel, self).__init__()
+        self.backbone = backbone
+        self.classifier = classifier
+        self.aux_classifier = aux_classifier  # Name aux is already take for COCO pre-trained models
+        self.lane_classifier = lane_classifier
+        self.channel_reducer = channel_reducer  # Reduce ResNet feature channels to 128 as did in RESA
+        self.scnn_layer = scnn_layer
+        self.recon_head = recon_head
+
+    def forward(self, x):
+        # input_shape = x.shape[-2:]
+        # contract: features is a dict of tensors
+        features = self.backbone(x)
+        result = OrderedDict()
+        x = features['out']
+
+        # For lane detection
+        if self.channel_reducer is not None:
+            x = self.channel_reducer(x)
+        if self.scnn_layer is not None:
+            x = self.scnn_layer(x)
+
+        # Semantic segmentation
+        x = self.classifier(x)
+        # x = F.interpolate(x, size=(513, 513), mode='bilinear', align_corners=True)
+        result['out'] = x
+
+        # For lane detection
+        if self.lane_classifier is not None:
+            result['lane'] = self.lane_classifier(x.softmax(dim=1))
+
+        # For COCO pre-trained models
+        if self.aux_classifier is not None:
+            x = features['aux']
+            x = self.aux_classifier(x)
+            # x = F.interpolate(x, size=input_shape, mode='bilinear', align_corners=True)
+            result['aux'] = x
+
+        return result

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/segmentation/deeplab.py

@@ -0,0 +1,120 @@
+# Note that this file only implement DeepLabs with ResNet backbone,
+# For the original DeepLab-LargeFOV with VGG backbone, refer to deeplab_vgg.py
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+from ..builder import MODELS
+
+
+@MODELS.register()
+class DeepLabV3Head(nn.Sequential):
+    def __init__(self, in_channels, num_classes):
+        super(DeepLabV3Head, self).__init__(
+            ASPP(in_channels, [12, 24, 36]),
+            nn.Conv2d(256, 256, 3, padding=1, bias=False),
+            nn.BatchNorm2d(256),
+            nn.ReLU(),
+            nn.Conv2d(256, num_classes, 1)
+        )
+
+
+# For better format consistency
+@MODELS.register()
+class DeepLabV2Head(nn.Sequential):
+    def __init__(self, in_channels, num_classes):
+        super(DeepLabV2Head, self).__init__(
+            ASPP_V2(in_channels, num_classes, [6, 12, 18, 24])
+        )
+
+
+# For better format consistency
+# Not the official VGG backbone version
+@MODELS.register()
+class DeepLabV1Head(nn.Sequential):
+    def __init__(self, in_channels, num_classes, dilation=12):
+        super(DeepLabV1Head, self).__init__(
+            LargeFOV(in_channels, num_classes, dilation)
+        )
+
+
+class ASPPConv(nn.Sequential):
+    def __init__(self, in_channels, out_channels, dilation):
+        modules = [
+            nn.Conv2d(in_channels, out_channels, 3, padding=dilation, dilation=dilation, bias=False),
+            nn.BatchNorm2d(out_channels),
+            nn.ReLU()
+        ]
+        super(ASPPConv, self).__init__(*modules)
+
+
+class ASPPPooling(nn.Sequential):
+    def __init__(self, in_channels, out_channels):
+        super(ASPPPooling, self).__init__(
+            nn.AdaptiveAvgPool2d(1),
+            nn.Conv2d(in_channels, out_channels, 1, bias=False),
+            nn.BatchNorm2d(out_channels),
+            nn.ReLU())
+
+    def forward(self, x):
+        size = x.shape[-2:]
+        x = super(ASPPPooling, self).forward(x)
+        return F.interpolate(x, size=size, mode='bilinear', align_corners=True)
+
+
+class ASPP(nn.Module):
+    def __init__(self, in_channels, atrous_rates):
+        super(ASPP, self).__init__()
+        out_channels = 256
+        modules = []
+        modules.append(nn.Sequential(
+            nn.Conv2d(in_channels, out_channels, 1, bias=False),
+            nn.BatchNorm2d(out_channels),
+            nn.ReLU()))
+
+        rate1, rate2, rate3 = tuple(atrous_rates)
+        modules.append(ASPPConv(in_channels, out_channels, rate1))
+        modules.append(ASPPConv(in_channels, out_channels, rate2))
+        modules.append(ASPPConv(in_channels, out_channels, rate3))
+        modules.append(ASPPPooling(in_channels, out_channels))
+
+        self.convs = nn.ModuleList(modules)
+
+        self.project = nn.Sequential(
+            nn.Conv2d(5 * out_channels, out_channels, 1, bias=False),
+            nn.BatchNorm2d(out_channels),
+            nn.ReLU(),
+            nn.Dropout(0.5))
+
+    def forward(self, x):
+        res = []
+        for conv in self.convs:
+            res.append(conv(x))
+        res = torch.cat(res, dim=1)
+        return self.project(res)
+
+
+# Copied and modified from hfslyc/AdvSemiSeg/model
+class ASPP_V2(nn.Module):
+    def __init__(self, in_channels, num_classes, atrous_rates):
+        super(ASPP_V2, self).__init__()
+        self.convs = nn.ModuleList()
+        for rates in atrous_rates:
+            self.convs.append(
+                nn.Conv2d(in_channels, num_classes, kernel_size=3, stride=1, padding=rates, dilation=rates, bias=True))
+
+    def forward(self, x):
+        res = self.convs[0](x)
+        for i in range(len(self.convs) - 1):
+            res += self.convs[i + 1](x)
+            return res
+
+
+class LargeFOV(nn.Module):
+    def __init__(self, in_channels, num_classes, dilation=12):
+        super(LargeFOV, self).__init__()
+        self.conv = nn.Conv2d(in_channels, num_classes, kernel_size=3, stride=1,
+                              padding=dilation, dilation=dilation, bias=True)
+
+    def forward(self, x):
+        return self.conv(x)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/segmentation/deeplab_vgg.py

@@ -0,0 +1,103 @@
+# Referenced from jcdubron/scnn_pytorch
+# DeepLabV1 and variants
+import torch.nn as nn
+from collections import OrderedDict
+
+from ..builder import MODELS
+
+
+@MODELS.register()
+class DeepLabV1(nn.Module):
+    # Original DeepLabV1 with VGG-16 for lane detection
+    def __init__(self, backbone_cfg,
+                 num_classes,
+                 spatial_conv_cfg=None,
+                 lane_classifier_cfg=None,
+                 dropout_1=0.1):
+        super(DeepLabV1, self).__init__()
+
+        self.encoder = MODELS.from_dict(backbone_cfg)
+        self.fc67 = nn.Sequential(
+            nn.Conv2d(512, 1024, 3, padding=4, dilation=4, bias=False),
+            nn.BatchNorm2d(1024),
+            nn.ReLU(),
+            nn.Conv2d(1024, 128, 1, bias=False),
+            nn.BatchNorm2d(128),
+            nn.ReLU()
+        )
+        self.scnn = MODELS.from_dict(spatial_conv_cfg)
+        self.fc8 = nn.Sequential(
+            nn.Dropout2d(dropout_1),
+            nn.Conv2d(128, num_classes, 1)
+        )
+        self.softmax = nn.Softmax(dim=1)
+        self.lane_classifier = MODELS.from_dict(lane_classifier_cfg)
+
+    def forward(self, x):
+        out = OrderedDict()
+
+        output = self.encoder(x)
+        output = self.fc67(output)
+
+        if self.scnn is not None:
+            output = self.scnn(output)
+
+        output = self.fc8(output)
+        out['out'] = output
+        if self.lane_classifier is not None:
+            output = self.softmax(output)
+            out['lane'] = self.lane_classifier(output)
+
+        return out
+
+
+@MODELS.register()
+class DeepLabV1Lane(nn.Module):
+    # General lane baseline
+    def __init__(self,
+                 backbone_cfg=None,
+                 spatial_conv_cfg=None,
+                 lane_classifier_cfg=None,
+                 reducer_cfg=None,
+                 classifier_cfg=None,
+                 uper_cfg=None):
+        super().__init__()
+        self.encoder = MODELS.from_dict(backbone_cfg)
+        self.reducer = MODELS.from_dict(reducer_cfg)
+        self.scnn = MODELS.from_dict(spatial_conv_cfg)
+        self.classifier = MODELS.from_dict(classifier_cfg)
+        self.softmax = nn.Softmax(dim=1)
+        self.lane_classifier = MODELS.from_dict(lane_classifier_cfg)
+        self.uper_decoder = MODELS.from_dict(uper_cfg)
+
+    def forward(self, input):
+        out = OrderedDict()
+        output = self.encoder(input)
+        if self.uper_decoder is not None:
+            output = self.uper_decoder(output)
+        if self.reducer is not None:
+            output = self.reducer(output)
+        if self.scnn is not None:
+            output = self.scnn(output)
+        output = self.classifier(output)
+        out['out'] = output
+        if self.lane_classifier is not None:
+            output = self.softmax(output)
+            out['lane'] = self.lane_classifier(output)
+        return out
+
+
+@MODELS.register()
+class SegRepVGG(DeepLabV1Lane):
+    def eval(self, profiling=False):
+        """A secure copy of eval()
+        """
+
+        if profiling:
+            for module in self.encoder.modules():
+                if hasattr(module, 'switch_to_deploy'):
+                    module.switch_to_deploy()
+            print('Deploy!')
+            return self.train(False)
+        else:
+            return self.train(False)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/segmentation/enet.py

@@ -0,0 +1,700 @@
+# copy from https://github.com/davidtvs/PyTorch-ENet/blob/master/models/enet.py
+from collections import OrderedDict
+import torch.nn as nn
+import torch
+from torch.nn import functional as F
+from torch.nn.parameter import Parameter
+
+from ._utils import _EncoderDecoderModel
+from ..builder import MODELS
+
+
+class InitialBlock(nn.Module):
+    """The initial block is composed of two branches:
+    1. a main branch which performs a regular convolution with stride 2;
+    2. an extension branch which performs max-pooling.
+    Doing both operations in parallel and concatenating their results
+    allows for efficient downsampling and expansion. The main branch
+    outputs 13 feature maps while the extension branch outputs 3, for a
+    total of 16 feature maps after concatenation.
+    Keyword arguments:
+    - in_channels (int): the number of input channels.
+    - out_channels (int): the number output channels.
+    - kernel_size (int, optional): the kernel size of the filters used in
+    the convolution layer. Default: 3.
+    - padding (int, optional): zero-padding added to both sides of the
+    input. Default: 0.
+    - bias (bool, optional): Adds a learnable bias to the output if
+    ``True``. Default: False.
+    - relu (bool, optional): When ``True`` ReLU is used as the activation
+    function; otherwise, PReLU is used. Default: True.
+    """
+
+    def __init__(self, in_channels, out_channels, bias=False, relu=True):
+        super().__init__()
+
+        if relu:
+            activation = nn.ReLU
+        else:
+            activation = nn.PReLU
+
+        # Main branch - As stated above the number of output channels for this
+        # branch is the total minus 3, since the remaining channels come from
+        # the extension branch
+        self.main_branch = nn.Conv2d(in_channels, out_channels - 3, kernel_size=3, stride=2, padding=1, bias=bias)
+
+        # Extension branch
+        self.ext_branch = nn.MaxPool2d(3, stride=2, padding=1)
+
+        # Initialize batch normalization to be used after concatenation
+        self.batch_norm = nn.BatchNorm2d(out_channels)
+
+        # PReLU layer to apply after concatenating the branches
+        self.out_activation = activation()
+
+    def forward(self, x):
+        main = self.main_branch(x)
+        ext = self.ext_branch(x)
+
+        # Concatenate branches
+        out = torch.cat((main, ext), 1)
+
+        # Apply batch normalization
+        out = self.batch_norm(out)
+
+        return self.out_activation(out)
+
+
+class RegularBottleneck(nn.Module):
+    """Regular bottlenecks are the main building block of ENet.
+    Main branch:
+    1. Shortcut connection.
+    Extension branch:
+    1. 1x1 convolution which decreases the number of channels by
+    ``internal_ratio``, also called a projection;
+    2. regular, dilated or asymmetric convolution;
+    3. 1x1 convolution which increases the number of channels back to
+    ``channels``, also called an expansion;
+    4. dropout as a regularizer.
+    Keyword arguments:
+    - channels (int): the number of input and output channels.
+    - internal_ratio (int, optional): a scale factor applied to
+    ``channels`` used to compute the number of
+    channels after the projection. eg. given ``channels`` equal to 128 and
+    internal_ratio equal to 2 the number of channels after the projection
+    is 64. Default: 4.
+    - kernel_size (int, optional): the kernel size of the filters used in
+    the convolution layer described above in item 2 of the extension
+    branch. Default: 3.
+    - padding (int, optional): zero-padding added to both sides of the
+    input. Default: 0.
+    - dilation (int, optional): spacing between kernel elements for the
+    convolution described in item 2 of the extension branch. Default: 1.
+    asymmetric (bool, optional): flags if the convolution described in
+    item 2 of the extension branch is asymmetric or not. Default: False.
+    - dropout_prob (float, optional): probability of an element to be
+    zeroed. Default: 0 (no dropout).
+    - bias (bool, optional): Adds a learnable bias to the output if
+    ``True``. Default: False.
+    - relu (bool, optional): When ``True`` ReLU is used as the activation
+    function; otherwise, PReLU is used. Default: True.
+    """
+
+    def __init__(self, channels, internal_ratio=4, kernel_size=3, padding=0, dilation=1, asymmetric=False,
+                 dropout_prob=0.0, is_dropout=True, bias=False, relu=True):
+        super().__init__()
+
+        # Check in the internal_scale parameter is within the expected range
+        # [1, channels]
+        if internal_ratio <= 1 or internal_ratio > channels:
+            raise RuntimeError("Value out of range. Expected value in the "
+                               "interval [1, {0}], got internal_scale={1}."
+                               .format(channels, internal_ratio))
+
+        internal_channels = channels // internal_ratio
+        self.is_dropout = is_dropout
+        if relu:
+            activation = nn.ReLU
+        else:
+            activation = nn.PReLU
+
+        # Main branch - shortcut connection
+
+        # Extension branch - 1x1 convolution, followed by a regular, dilated or
+        # asymmetric convolution, followed by another 1x1 convolution, and,
+        # finally, a regularizer (spatial dropout). Number of channels is constant.
+
+        # 1x1 projection convolution
+        self.ext_conv1 = nn.Sequential(
+            nn.Conv2d(channels, internal_channels, kernel_size=1, stride=1, bias=bias),
+            nn.BatchNorm2d(internal_channels),
+            activation()
+        )
+
+        # If the convolution is asymmetric we split the main convolution in
+        # two. Eg. for a 5x5 asymmetric convolution we have two convolution:
+        # the first is 5x1 and the second is 1x5.
+        if asymmetric:
+            self.ext_conv2 = nn.Sequential(
+                nn.Conv2d(internal_channels, internal_channels, kernel_size=(kernel_size, 1), stride=1,
+                          padding=(padding, 0), dilation=dilation, bias=bias),
+                nn.BatchNorm2d(internal_channels),
+                activation(),
+                nn.Conv2d(internal_channels, internal_channels, kernel_size=(1, kernel_size), stride=1,
+                          padding=(0, padding), dilation=dilation, bias=bias),
+                nn.BatchNorm2d(internal_channels),
+                activation())
+        else:
+            self.ext_conv2 = nn.Sequential(
+                nn.Conv2d(internal_channels, internal_channels, kernel_size=kernel_size, stride=1, padding=padding,
+                          dilation=dilation, bias=bias),
+                nn.BatchNorm2d(internal_channels),
+                activation())
+
+        # 1x1 expansion convolution
+        self.ext_conv3 = nn.Sequential(
+            nn.Conv2d(internal_channels, channels, kernel_size=1, stride=1, bias=bias),
+            nn.BatchNorm2d(channels),
+            # activation()
+        )
+
+        self.ext_regul = nn.Dropout2d(p=dropout_prob)
+
+        # PReLU layer to apply after adding the branches
+        self.out_activation = activation()
+
+    def forward(self, x):
+        # Main branch shortcut
+        main = x
+
+        # Extension branch
+        ext = self.ext_conv1(x)
+        ext = self.ext_conv2(ext)
+        ext = self.ext_conv3(ext)
+        if self.is_dropout:
+            ext = self.ext_regul(ext)
+
+        # Add main and extension branches
+        out = main + ext
+
+        return self.out_activation(out)
+
+
+class DownsamplingBottleneck(nn.Module):
+    """Downsampling bottlenecks further downsample the feature map size.
+    Main branch:
+    1. max pooling with stride 2; indices are saved to be used for
+    unpooling later.
+    Extension branch:
+    1. 2x2 convolution with stride 2 that decreases the number of channels
+    by ``internal_ratio``, also called a projection;
+    2. regular convolution (by default, 3x3);
+    3. 1x1 convolution which increases the number of channels to
+    ``out_channels``, also called an expansion;
+    4. dropout as a regularizer.
+    Keyword arguments:
+    - in_channels (int): the number of input channels.
+    - out_channels (int): the number of output channels.
+    - internal_ratio (int, optional): a scale factor applied to ``channels``
+    used to compute the number of channels after the projection. eg. given
+    ``channels`` equal to 128 and internal_ratio equal to 2 the number of
+    channels after the projection is 64. Default: 4.
+    - return_indices (bool, optional):  if ``True``, will return the max
+    indices along with the outputs. Useful when unpooling later.
+    - dropout_prob (float, optional): probability of an element to be
+    zeroed. Default: 0 (no dropout).
+    - bias (bool, optional): Adds a learnable bias to the output if
+    ``True``. Default: False.
+    - relu (bool, optional): When ``True`` ReLU is used as the activation
+    function; otherwise, PReLU is used. Default: True.
+    """
+
+    def __init__(self, in_channels, out_channels, internal_ratio=4, return_indices=False, dropout_prob=0.0,
+                 is_dropout=True, bias=False,
+                 relu=True):
+        super().__init__()
+
+        # Store parameters that are needed later
+        self.return_indices = return_indices
+        self.is_dropout = is_dropout
+        # Check in the internal_scale parameter is within the expected range
+        # [1, channels]
+        if internal_ratio <= 1 or internal_ratio > in_channels:
+            raise RuntimeError("Value out of range. Expected value in the "
+                               "interval [1, {0}], got internal_scale={1}. "
+                               .format(in_channels, internal_ratio))
+
+        internal_channels = in_channels // internal_ratio
+
+        if relu:
+            activation = nn.ReLU
+        else:
+            activation = nn.PReLU
+
+        # Main branch - max pooling followed by feature map (channels) padding
+        self.main_max1 = nn.MaxPool2d(2, stride=2, return_indices=return_indices)
+
+        # Extension branch - 2x2 convolution, followed by a regular, dilated or
+        # asymmetric convolution, followed by another 1x1 convolution. Number
+        # of channels is doubled.
+
+        # 2x2 projection convolution with stride 2
+        self.ext_conv1 = nn.Sequential(
+            nn.Conv2d(in_channels, internal_channels, kernel_size=2, stride=2, bias=bias),
+            nn.BatchNorm2d(internal_channels),
+            activation())
+
+        # Convolution
+        self.ext_conv2 = nn.Sequential(
+            nn.Conv2d(internal_channels, internal_channels, kernel_size=3, stride=1, padding=1, bias=bias),
+            nn.BatchNorm2d(internal_channels),
+            activation())
+
+        # 1x1 expansion convolution
+        self.ext_conv3 = nn.Sequential(
+            nn.Conv2d(internal_channels, out_channels, kernel_size=1, stride=1, bias=bias),
+            nn.BatchNorm2d(out_channels),
+            # activation()
+        )
+
+        self.ext_regul = nn.Dropout2d(p=dropout_prob)
+
+        # PReLU layer to apply after concatenating the branches
+        self.out_activation = activation()
+
+    def forward(self, x):
+        # Main branch shortcut
+        if self.return_indices:
+            main, max_indices = self.main_max1(x)
+        else:
+            main = self.main_max1(x)
+
+        # Extension branch
+        ext = self.ext_conv1(x)
+        ext = self.ext_conv2(ext)
+        ext = self.ext_conv3(ext)
+        if self.is_dropout:
+            ext = self.ext_regul(ext)
+
+        # Main branch channel padding
+        n, ch_ext, h, w = ext.size()
+        ch_main = main.size()[1]
+        padding = torch.zeros(n, ch_ext - ch_main, h, w)
+
+        # Before concatenating, check if main is on the CPU or GPU and
+        # convert padding accordingly
+        if main.is_cuda:
+            padding = padding.cuda()
+
+        # Concatenate
+        main = torch.cat((main, padding), 1)
+
+        # Add main and extension branches
+        out = main + ext
+
+        return self.out_activation(out), max_indices
+
+
+class UpsamplingBottleneck(nn.Module):
+    """The upsampling bottlenecks upsample the feature map resolution using max
+    pooling indices stored from the corresponding downsampling bottleneck.
+    Main branch:
+    1. 1x1 convolution with stride 1 that decreases the number of channels by
+    ``internal_ratio``, also called a projection;
+    2. max unpool layer using the max pool indices from the corresponding
+    downsampling max pool layer.
+    Extension branch:
+    1. 1x1 convolution with stride 1 that decreases the number of channels by
+    ``internal_ratio``, also called a projection;
+    2. transposed convolution (by default, 3x3);
+    3. 1x1 convolution which increases the number of channels to
+    ``out_channels``, also called an expansion;
+    4. dropout as a regularizer.
+    Keyword arguments:
+    - in_channels (int): the number of input channels.
+    - out_channels (int): the number of output channels.
+    - internal_ratio (int, optional): a scale factor applied to ``in_channels``
+     used to compute the number of channels after the projection. eg. given
+     ``in_channels`` equal to 128 and ``internal_ratio`` equal to 2 the number
+     of channels after the projection is 64. Default: 4.
+    - dropout_prob (float, optional): probability of an element to be zeroed.
+    Default: 0 (no dropout).
+    - bias (bool, optional): Adds a learnable bias to the output if ``True``.
+    Default: False.
+    - relu (bool, optional): When ``True`` ReLU is used as the activation
+    function; otherwise, PReLU is used. Default: True.
+    """
+
+    def __init__(self, in_channels, out_channels, internal_ratio=4, dropout_prob=0.0, is_dropout=True, bias=False,
+                 relu=True):
+        super().__init__()
+
+        # Check in the internal_scale parameter is within the expected range
+        # [1, channels]
+        if internal_ratio <= 1 or internal_ratio > in_channels:
+            raise RuntimeError("Value out of range. Expected value in the "
+                               "interval [1, {0}], got internal_scale={1}. "
+                               .format(in_channels, internal_ratio))
+
+        internal_channels = in_channels // internal_ratio
+        self.is_dropout = is_dropout
+        if relu:
+            activation = nn.ReLU
+        else:
+            activation = nn.PReLU
+
+        # Main branch - max pooling followed by feature map (channels) padding
+        self.main_conv1 = nn.Sequential(
+            nn.Conv2d(in_channels, out_channels, kernel_size=1, bias=bias),
+            nn.BatchNorm2d(out_channels))
+
+        # Remember that the stride is the same as the kernel_size, just like
+        # the max pooling layers
+        self.main_unpool1 = nn.MaxUnpool2d(kernel_size=2)
+
+        # Extension branch - 1x1 convolution, followed by a regular, dilated or
+        # asymmetric convolution, followed by another 1x1 convolution. Number
+        # of channels is doubled.
+
+        # 1x1 projection convolution with stride 1
+        self.ext_conv1 = nn.Sequential(
+            nn.Conv2d(in_channels, internal_channels, kernel_size=1, bias=bias),
+            nn.BatchNorm2d(internal_channels), activation())
+
+        # Transposed convolution
+        self.ext_tconv1 = nn.ConvTranspose2d(internal_channels, internal_channels, kernel_size=2, stride=2, bias=bias)
+        self.ext_tconv1_bnorm = nn.BatchNorm2d(internal_channels)
+        self.ext_tconv1_activation = activation()
+
+        # 1x1 expansion convolution
+        self.ext_conv2 = nn.Sequential(
+            nn.Conv2d(internal_channels, out_channels, kernel_size=1, bias=bias),
+            nn.BatchNorm2d(out_channels),
+            # activation()
+        )
+
+        self.ext_regul = nn.Dropout2d(p=dropout_prob)
+
+        # PReLU layer to apply after concatenating the branches
+        self.out_activation = activation()
+
+    def forward(self, x, max_indices, output_size):
+        # Main branch shortcut
+        main = self.main_conv1(x)
+        main = self.main_unpool1(main, max_indices, output_size=output_size)
+
+        # Extension branch
+        ext = self.ext_conv1(x)
+        ext = self.ext_tconv1(ext, output_size=output_size)
+        ext = self.ext_tconv1_bnorm(ext)
+        ext = self.ext_tconv1_activation(ext)
+        ext = self.ext_conv2(ext)
+        if self.is_dropout:
+            ext = self.ext_regul(ext)
+
+        # Add main and extension branches
+        out = main + ext
+
+        return self.out_activation(out)
+
+
+class SpatialSoftmax(nn.Module):
+    def __init__(self, temperature=1, device='cpu'):
+        super(SpatialSoftmax, self).__init__()
+
+        if temperature:
+            self.temperature = Parameter(torch.ones(1) * temperature)
+        else:
+            self.temperature = 1.
+
+    def forward(self, feature):
+        feature = feature.view(feature.shape[0], -1, feature.shape[1] * feature.shape[2])
+        softmax_attention = F.softmax(feature / self.temperature, dim=-1)
+
+        return softmax_attention
+
+
+# class LaneExistENet(nn.Module):
+#     def __init__(self, num_output, num_lane, dropout=0.1, flattened_size=4500):
+#         super().__init__()
+#         self.conv_dilated = nn.Conv2d(128, 32, kernel_size=3, stride=1, padding=4, dilation=4)
+#         self.bn = nn.BatchNorm2d(32)
+#         self.dropout = nn.Dropout2d(dropout)
+#         self.conv1x1 = nn.Conv2d(32, num_lane, kernel_size=1, stride=1)
+#         self.ssm = SpatialSoftmax()
+#         self.avgpool = nn.AvgPool2d(2, 2)
+#         self.linear1 = nn.Linear(flattened_size, 128)
+#         self.linear2 = nn.Linear(128, num_output)
+#         self.sigmoid = nn.Sigmoid()
+#
+#     def forward(self, input):
+#         # input: logits
+#         output = self.conv_dilated(input)
+#         output = self.bn(output)
+#         output = F.relu(output)
+#         output = self.ssm(output)
+#         output = self.avgpool(output)
+#         output = output.flatten(start_dim=1)
+#         output = self.linear1(output)
+#         output = F.relu(output)
+#         output = self.linear2(output)
+#         output = self.sigmoid(output)
+#         return output
+
+
+# class ENet(nn.Module):
+#     """Generate the ENet model.
+#     Keyword arguments:
+#     - num_classes (int): the number of classes to segment.
+#     - encoder_relu (bool, optional): When ``True`` ReLU is used as the
+#     activation function in the encoder blocks/layers; otherwise, PReLU
+#     is used. Default: False.
+#     - decoder_relu (bool, optional): When ``True`` ReLU is used as the
+#     activation function in the decoder blocks/layers; otherwise, PReLU
+#     is used. Default: True.
+#     """
+#
+#     def __init__(self, num_classes, encoder_relu=False, decoder_relu=True, dropout_1=0.01, dropout_2=0.1):
+#         super().__init__()
+#
+#         self.initial_block = InitialBlock(3, 16, relu=encoder_relu)
+#
+#         # Stage 1 - Encoder
+#         self.downsample1_0 = DownsamplingBottleneck(16, 64, return_indices=True, dropout_prob=dropout_1,
+#                                                     relu=encoder_relu)
+#         self.regular1_1 = RegularBottleneck(64, padding=1, dropout_prob=dropout_1, relu=encoder_relu)
+#         self.regular1_2 = RegularBottleneck(64, padding=1, dropout_prob=dropout_1, relu=encoder_relu)
+#         self.regular1_3 = RegularBottleneck(64, padding=1, dropout_prob=dropout_1, relu=encoder_relu)
+#         self.regular1_4 = RegularBottleneck(64, padding=1, dropout_prob=dropout_1, relu=encoder_relu)
+#
+#         # Stage 2 - Encoder
+#         self.downsample2_0 = DownsamplingBottleneck(64, 128, return_indices=True, dropout_prob=dropout_2,
+#                                                     relu=encoder_relu)
+#         self.regular2_1 = RegularBottleneck(128, padding=1, dropout_prob=dropout_2, relu=encoder_relu)
+#         self.dilated2_2 = RegularBottleneck(128, dilation=2, padding=2, dropout_prob=dropout_2, relu=encoder_relu)
+#         self.asymmetric2_3 = RegularBottleneck(128, kernel_size=5, padding=2, asymmetric=True, dropout_prob=dropout_2,
+#                                                relu=encoder_relu)
+#         self.dilated2_4 = RegularBottleneck(128, dilation=4, padding=4, dropout_prob=dropout_2, relu=encoder_relu)
+#         self.regular2_5 = RegularBottleneck(128, padding=1, dropout_prob=dropout_2, relu=encoder_relu)
+#         self.dilated2_6 = RegularBottleneck(128, dilation=8, padding=8, dropout_prob=dropout_2, relu=encoder_relu)
+#         self.asymmetric2_7 = RegularBottleneck(128, kernel_size=5, asymmetric=True, padding=2, dropout_prob=dropout_2,
+#                                                relu=encoder_relu)
+#         self.dilated2_8 = RegularBottleneck(128, dilation=16, padding=16, dropout_prob=dropout_2, relu=encoder_relu)
+#
+#         # Stage 3 - Encoder
+#         self.regular3_0 = RegularBottleneck(128, padding=1, dropout_prob=dropout_2, relu=encoder_relu)
+#         self.dilated3_1 = RegularBottleneck(128, dilation=2, padding=2, dropout_prob=dropout_2, relu=encoder_relu)
+#         self.asymmetric3_2 = RegularBottleneck(128, kernel_size=5, padding=2, asymmetric=True, dropout_prob=dropout_2,
+#                                                relu=encoder_relu)
+#         self.dilated3_3 = RegularBottleneck(128, dilation=4, padding=4, dropout_prob=dropout_2, relu=encoder_relu)
+#         self.regular3_4 = RegularBottleneck(128, padding=1, dropout_prob=dropout_2, relu=encoder_relu)
+#         self.dilated3_5 = RegularBottleneck(128, dilation=8, padding=8, dropout_prob=dropout_2, relu=encoder_relu)
+#         self.asymmetric3_6 = RegularBottleneck(128, kernel_size=5, asymmetric=True, padding=2, dropout_prob=dropout_2,
+#                                                relu=encoder_relu)
+#         self.dilated3_7 = RegularBottleneck(128, dilation=16, padding=16, dropout_prob=dropout_2, relu=encoder_relu)
+#
+#         # Stage 4 - Decoder
+#         self.upsample4_0 = UpsamplingBottleneck(128, 64, dropout_prob=dropout_2, is_dropout=False, relu=decoder_relu)
+#         self.regular4_1 = RegularBottleneck(64, padding=1, dropout_prob=dropout_2, is_dropout=False, relu=decoder_relu)
+#         self.regular4_2 = RegularBottleneck(64, padding=1, dropout_prob=dropout_2, is_dropout=False, relu=decoder_relu)
+#
+#         # Stage 5 - Decoder
+#         self.upsample5_0 = UpsamplingBottleneck(64, 16, dropout_prob=dropout_2, is_dropout=False, relu=decoder_relu)
+#         self.regular5_1 = RegularBottleneck(16, padding=1, dropout_prob=dropout_2, is_dropout=False, relu=decoder_relu)
+#         self.transposed_conv = nn.ConvTranspose2d(16, num_classes, kernel_size=3, stride=2, padding=1, bias=False)
+#
+#     def forward(self, x):
+#         # Initial block
+#         input_size = x.size()
+#         x = self.initial_block(x)
+#
+#         # Stage 1 - Encoder
+#         stage1_input_size = x.size()
+#         x, max_indices1_0 = self.downsample1_0(x)
+#         x = self.regular1_1(x)
+#         x = self.regular1_2(x)
+#         x = self.regular1_3(x)
+#         x = self.regular1_4(x)
+#
+#         # Stage 2 - Encoder
+#         stage2_input_size = x.size()
+#         x, max_indices2_0 = self.downsample2_0(x)
+#         x = self.regular2_1(x)
+#         x = self.dilated2_2(x)
+#         x = self.asymmetric2_3(x)
+#         x = self.dilated2_4(x)
+#         x = self.regular2_5(x)
+#         x = self.dilated2_6(x)
+#         x = self.asymmetric2_7(x)
+#         x = self.dilated2_8(x)
+#
+#         # Stage 3 - Encoder
+#         x = self.regular3_0(x)
+#         x = self.dilated3_1(x)
+#         x = self.asymmetric3_2(x)
+#         x = self.dilated3_3(x)
+#         x = self.regular3_4(x)
+#         x = self.dilated3_5(x)
+#         x = self.asymmetric3_6(x)
+#         x = self.dilated3_7(x)
+#
+#         # Stage 4 - Decoder
+#         x = self.upsample4_0(x, max_indices2_0, output_size=stage2_input_size)
+#         x = self.regular4_1(x)
+#         x = self.regular4_2(x)
+#
+#         # Stage 5 - Decoder
+#         x = self.upsample5_0(x, max_indices1_0, output_size=stage1_input_size)
+#         x = self.regular5_1(x)
+#         x = self.transposed_conv(x, output_size=input_size)
+#
+#         return x
+
+
+class Encoder(nn.Module):
+
+    def __init__(self, encoder_relu=False, dropout_1=0.01, dropout_2=0.1):
+        super().__init__()
+
+        self.initial_block = InitialBlock(3, 16, relu=encoder_relu)
+
+        # Stage 1 - Encoder
+        self.downsample1_0 = DownsamplingBottleneck(16, 64, return_indices=True, dropout_prob=dropout_1,
+                                                    relu=encoder_relu)
+        self.regular1_1 = RegularBottleneck(64, padding=1, dropout_prob=dropout_1, relu=encoder_relu)
+        self.regular1_2 = RegularBottleneck(64, padding=1, dropout_prob=dropout_1, relu=encoder_relu)
+        self.regular1_3 = RegularBottleneck(64, padding=1, dropout_prob=dropout_1, relu=encoder_relu)
+        self.regular1_4 = RegularBottleneck(64, padding=1, dropout_prob=dropout_1, relu=encoder_relu)
+
+        # Stage 2 - Encoder
+        self.downsample2_0 = DownsamplingBottleneck(64, 128, return_indices=True, dropout_prob=dropout_2,
+                                                    relu=encoder_relu)
+        self.regular2_1 = RegularBottleneck(128, padding=1, dropout_prob=dropout_2, relu=encoder_relu)
+        self.dilated2_2 = RegularBottleneck(128, dilation=2, padding=2, dropout_prob=dropout_2, relu=encoder_relu)
+        self.asymmetric2_3 = RegularBottleneck(128, kernel_size=5, padding=2, asymmetric=True, dropout_prob=dropout_2,
+                                               relu=encoder_relu)
+        self.dilated2_4 = RegularBottleneck(128, dilation=4, padding=4, dropout_prob=dropout_2, relu=encoder_relu)
+        self.regular2_5 = RegularBottleneck(128, padding=1, dropout_prob=dropout_2, relu=encoder_relu)
+        self.dilated2_6 = RegularBottleneck(128, dilation=8, padding=8, dropout_prob=dropout_2, relu=encoder_relu)
+        self.asymmetric2_7 = RegularBottleneck(128, kernel_size=5, asymmetric=True, padding=2, dropout_prob=dropout_2,
+                                               relu=encoder_relu)
+        self.dilated2_8 = RegularBottleneck(128, dilation=16, padding=16, dropout_prob=dropout_2, relu=encoder_relu)
+
+        # Stage 3 - Encoder
+        self.regular3_0 = RegularBottleneck(128, padding=1, dropout_prob=dropout_2, relu=encoder_relu)
+        self.dilated3_1 = RegularBottleneck(128, dilation=2, padding=2, dropout_prob=dropout_2, relu=encoder_relu)
+        self.asymmetric3_2 = RegularBottleneck(128, kernel_size=5, padding=2, asymmetric=True, dropout_prob=dropout_2,
+                                               relu=encoder_relu)
+        self.dilated3_3 = RegularBottleneck(128, dilation=4, padding=4, dropout_prob=dropout_2, relu=encoder_relu)
+        self.regular3_4 = RegularBottleneck(128, padding=1, dropout_prob=dropout_2, relu=encoder_relu)
+        self.dilated3_5 = RegularBottleneck(128, dilation=8, padding=8, dropout_prob=dropout_2, relu=encoder_relu)
+        self.asymmetric3_6 = RegularBottleneck(128, kernel_size=5, asymmetric=True, padding=2, dropout_prob=dropout_2,
+                                               relu=encoder_relu)
+        self.dilated3_7 = RegularBottleneck(128, dilation=16, padding=16, dropout_prob=dropout_2, relu=encoder_relu)
+
+    def forward(self, x):
+        # Initial block
+        input_size = x.size()
+        x = self.initial_block(x)
+
+        # Stage 1 - Encoder
+        stage1_input_size = x.size()
+        x, max_indices1_0 = self.downsample1_0(x)
+        x = self.regular1_1(x)
+        x = self.regular1_2(x)
+        x = self.regular1_3(x)
+        x = self.regular1_4(x)
+
+        # Stage 2 - Encoder
+        stage2_input_size = x.size()
+        x, max_indices2_0 = self.downsample2_0(x)
+        x = self.regular2_1(x)
+        x = self.dilated2_2(x)
+        x = self.asymmetric2_3(x)
+        x = self.dilated2_4(x)
+        x = self.regular2_5(x)
+        x = self.dilated2_6(x)
+        x = self.asymmetric2_7(x)
+        x = self.dilated2_8(x)
+
+        # Stage 3 - Encoder
+        x = self.regular3_0(x)
+        x = self.dilated3_1(x)
+        x = self.asymmetric3_2(x)
+        x = self.dilated3_3(x)
+        x = self.regular3_4(x)
+        x = self.dilated3_5(x)
+        x = self.asymmetric3_6(x)
+        x = self.dilated3_7(x)
+
+        return x, max_indices1_0, stage1_input_size, max_indices2_0, stage2_input_size, input_size
+
+
+class Decoder(nn.Module):
+
+    def __init__(self, num_classes, decoder_relu=True, dropout_2=0.1):
+        super().__init__()
+
+        # Stage 4 - Decoder
+        self.upsample4_0 = UpsamplingBottleneck(128, 64, dropout_prob=dropout_2, is_dropout=False, relu=decoder_relu)
+        self.regular4_1 = RegularBottleneck(64, padding=1, dropout_prob=dropout_2, is_dropout=False, relu=decoder_relu)
+        self.regular4_2 = RegularBottleneck(64, padding=1, dropout_prob=dropout_2, is_dropout=False, relu=decoder_relu)
+
+        # Stage 5 - Decoder
+        self.upsample5_0 = UpsamplingBottleneck(64, 16, dropout_prob=dropout_2, is_dropout=False, relu=decoder_relu)
+        self.regular5_1 = RegularBottleneck(16, padding=1, dropout_prob=dropout_2, is_dropout=False, relu=decoder_relu)
+        self.transposed_conv = nn.ConvTranspose2d(16, num_classes, kernel_size=3, stride=2, padding=1, bias=False)
+
+    def forward(self, x, max_indices1_0, stage1_input_size, max_indices2_0, stage2_input_size, input_size):
+        # Stage 4 - Decoder
+        x = self.upsample4_0(x, max_indices2_0, output_size=stage2_input_size)
+        x = self.regular4_1(x)
+        x = self.regular4_2(x)
+
+        # Stage 5 - Decoder
+        x = self.upsample5_0(x, max_indices1_0, output_size=stage1_input_size)
+        x = self.regular5_1(x)
+        x = self.transposed_conv(x, output_size=input_size)
+
+        return x
+
+
+@MODELS.register()
+class ENet(_EncoderDecoderModel):
+    def __init__(self,
+                 num_classes,
+                 lane_classifier_cfg=None,
+                 encoder_relu=False,
+                 decoder_relu=True,
+                 dropout_1=0.01,
+                 dropout_2=0.1,
+                 encoder_only=False,
+                 pretrained_weights=None):
+        super().__init__()
+        self.encoder_conv = None
+        self.encoder = Encoder(encoder_relu=encoder_relu, dropout_1=dropout_1, dropout_2=dropout_2)
+
+        if encoder_only:
+            self.decoder = None
+            self.encoder_conv = nn.Conv2d(128, num_classes, kernel_size=1, stride=1)
+        else:
+            self.decoder = Decoder(num_classes=num_classes, decoder_relu=decoder_relu, dropout_2=dropout_2)
+
+        self.lane_classifier = MODELS.from_dict(lane_classifier_cfg)
+        self._load_encoder(pretrained_weights)
+
+    def forward(self, x):
+        out = OrderedDict()
+        x, max_indices1_0, stage1_input_size, max_indices2_0, stage2_input_size, input_size = self.encoder(x)
+        if self.encoder_conv is not None:
+            x = self.encoder_conv(x)
+
+        if self.lane_classifier is not None:
+            out['lane'] = self.lane_classifier(x)
+
+        if self.decoder is not None:
+            x = self.decoder.forward(x, max_indices1_0, stage1_input_size, max_indices2_0,
+                                     stage2_input_size, input_size)
+        out['out'] = x
+
+        return out

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/segmentation/erfnet.py

@@ -0,0 +1,144 @@
+# Copied and modified from Eromera/erfnet_pytorch,
+# cardwing/Codes-for-Lane-Detection and
+# jcdubron/scnn_pytorch
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from collections import OrderedDict
+
+from ..common_models import non_bottleneck_1d
+from ..builder import MODELS
+from ._utils import _EncoderDecoderModel
+
+
+class DownsamplerBlock(nn.Module):
+    def __init__(self, ninput, noutput):
+        super().__init__()
+        self.conv = nn.Conv2d(ninput, noutput-ninput, (3, 3), stride=2, padding=1, bias=True)
+        self.pool = nn.MaxPool2d(2, stride=2)
+        self.bn = nn.BatchNorm2d(noutput, eps=1e-3)
+
+    def forward(self, input):
+        output = torch.cat([self.conv(input), self.pool(input)], 1)
+        output = self.bn(output)
+        return F.relu(output)
+
+
+class Encoder(nn.Module):
+    def __init__(self, num_classes, dropout_1=0.03, dropout_2=0.3):
+        super().__init__()
+        self.initial_block = DownsamplerBlock(3, 16)
+
+        self.layers = nn.ModuleList()
+
+        self.layers.append(DownsamplerBlock(16, 64))
+
+        for x in range(0, 5):  # 5 times
+            self.layers.append(non_bottleneck_1d(64, dropout_1, 1))
+
+        self.layers.append(DownsamplerBlock(64, 128))
+
+        for x in range(0, 2):  # 2 times
+            self.layers.append(non_bottleneck_1d(128, dropout_2, 2))
+            self.layers.append(non_bottleneck_1d(128, dropout_2, 4))
+            self.layers.append(non_bottleneck_1d(128, dropout_2, 8))
+            self.layers.append(non_bottleneck_1d(128, dropout_2, 16))
+
+        # Only in encoder mode:
+        self.output_conv = nn.Conv2d(128, num_classes, 1, stride=1, padding=0, bias=True)
+
+    def forward(self, input, predict=False):
+        output = self.initial_block(input)
+
+        for layer in self.layers:
+            output = layer(output)
+
+        if predict:
+            output = self.output_conv(output)
+
+        return output
+
+
+class UpsamplerBlock(nn.Module):
+    def __init__(self, ninput, noutput):
+        super().__init__()
+        self.conv = nn.ConvTranspose2d(ninput, noutput, 3, stride=2, padding=1, output_padding=1, bias=True)
+        self.bn = nn.BatchNorm2d(noutput, eps=1e-3)
+
+    def forward(self, input):
+        output = self.conv(input)
+        output = self.bn(output)
+        return F.relu(output)
+
+
+class Decoder(nn.Module):
+    def __init__(self, num_classes):
+        super().__init__()
+        self.layers = nn.ModuleList()
+        self.layers.append(UpsamplerBlock(128, 64))
+        self.layers.append(non_bottleneck_1d(64, 0, 1))
+        self.layers.append(non_bottleneck_1d(64, 0, 1))
+        self.layers.append(UpsamplerBlock(64, 16))
+        self.layers.append(non_bottleneck_1d(16, 0, 1))
+        self.layers.append(non_bottleneck_1d(16, 0, 1))
+
+        self.output_conv = nn.ConvTranspose2d(16, num_classes, 2, stride=2, padding=0, output_padding=0, bias=True)
+
+    def forward(self, input):
+        output = input
+
+        for layer in self.layers:
+            output = layer(output)
+
+        output = self.output_conv(output)
+
+        return output
+
+
+# ERFNet
+@MODELS.register()
+class ERFNet(_EncoderDecoderModel):
+    def __init__(self,
+                 num_classes,
+                 lane_classifier_cfg=None,
+                 spatial_conv_cfg=None,
+                 dropout_1=0.03,
+                 dropout_2=0.3,
+                 pretrained_weights=None):
+        super().__init__()
+        self.encoder = Encoder(num_classes=num_classes, dropout_1=dropout_1, dropout_2=dropout_2)
+        self.decoder = Decoder(num_classes)
+        self.spatial_conv = MODELS.from_dict(spatial_conv_cfg)
+        self.lane_classifier = MODELS.from_dict(lane_classifier_cfg)
+        self._load_encoder(pretrained_weights)
+
+    def _load_encoder(self, pretrained_weights):
+        if pretrained_weights is not None:  # Load ImageNet pre-trained weights
+            try:
+                saved_weights = torch.load(pretrained_weights)['state_dict']
+            except FileNotFoundError:
+                raise FileNotFoundError('pretrained_weights is not there! '
+                                        'Please set pretrained_weights=None if you are only testing.')
+            original_weights = self.state_dict()
+            for key in saved_weights.keys():
+                my_key = key.replace('module.features.', '')
+                if my_key in original_weights.keys():
+                    original_weights[my_key] = saved_weights[key]
+            self.load_state_dict(original_weights)
+        else:
+            print('No ImageNet pre-training.')
+
+    def forward(self, x, only_encode=False):
+        out = OrderedDict()
+        if only_encode:
+            return self.encoder.forward(x, predict=True)
+        else:
+            output = self.encoder(x)    # predict=False by default
+            if self.spatial_conv is not None:
+                output = self.spatial_conv(output)
+            out['out'] = self.decoder.forward(output)
+
+            if self.lane_classifier is not None:
+                out['lane'] = self.lane_classifier(output)
+            return out

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/segmentation/fcn.py

@@ -0,0 +1,18 @@
+from torch import nn
+
+from ..builder import MODELS
+
+
+@MODELS.register()
+class FCNHead(nn.Sequential):
+    def __init__(self, in_channels, num_classes):
+        inter_channels = in_channels // 4
+        layers = [
+            nn.Conv2d(in_channels, inter_channels, 3, padding=1, bias=False),
+            nn.BatchNorm2d(inter_channels),
+            nn.ReLU(),
+            nn.Dropout(0.1),
+            nn.Conv2d(inter_channels, num_classes, 1)
+        ]
+
+        super(FCNHead, self).__init__(*layers)

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/segmentation/segmentation.py

@@ -0,0 +1,59 @@
+from ..utils import load_state_dict_from_url
+from ._utils import _SimpleSegmentationModel
+from ..builder import MODELS
+
+model_urls = {
+    'fcn_resnet50_coco': None,
+    'fcn_resnet101_coco': 'https://download.pytorch.org/models/fcn_resnet101_coco-7ecb50ca.pth',
+    'deeplabv3_resnet50_coco': None,
+    'deeplabv3_resnet101_coco': 'https://download.pytorch.org/models/deeplabv3_resnet101_coco-586e9e4e.pth',
+    'deeplabv2_resnet101_coco': None,
+    'deeplabv1_resnet18_coco': None,
+    'deeplabv1_resnet34_coco': None,
+    'deeplabv1_resnet50_coco': None,
+    'deeplabv1_resnet101_coco': None
+}
+
+
+def _build_segmentation_net(backbone_cfg, classifier_cfg,
+                            aux_classifier_cfg=None, lane_classifier_cfg=None, reducer_cfg=None, spatial_conv_cfg=None):
+    if aux_classifier_cfg is not None:  # Fixed for COCO pre-training
+        backbone_cfg['return_layer'] = {
+            'layer3': 'aux',
+            'layer4': 'out'
+        }
+    backbone = MODELS.from_dict(backbone_cfg)
+    classifier = MODELS.from_dict(classifier_cfg)
+    aux_classifier = MODELS.from_dict(aux_classifier_cfg)
+    lane_classifier = MODELS.from_dict(lane_classifier_cfg)
+    channel_reducer = MODELS.from_dict(reducer_cfg)
+    scnn_layer = MODELS.from_dict(spatial_conv_cfg)
+
+    model = _SimpleSegmentationModel(
+        backbone, classifier, aux_classifier,
+        lane_classifier=lane_classifier, channel_reducer=channel_reducer, scnn_layer=scnn_layer)
+
+    return model
+
+
+@MODELS.register()
+def standard_segmentation_model(backbone_cfg, classifier_cfg,
+                                progress=True, pretrained=False, arch_type=None, backbone=None,
+                                aux_classifier_cfg=None,
+                                lane_classifier_cfg=None, reducer_cfg=None, spatial_conv_cfg=None):
+    if pretrained:
+        assert 'aux_classifier_cfg' is not None
+        assert arch_type is not None
+        assert backbone is not None
+    model = _build_segmentation_net(backbone_cfg, classifier_cfg,
+                                    aux_classifier_cfg, lane_classifier_cfg, reducer_cfg, spatial_conv_cfg)
+    if pretrained:
+        arch = arch_type + '_' + backbone + '_coco'
+        model_url = model_urls[arch]
+        if model_url is None:
+            raise NotImplementedError('COCO pretrained {} is not supported as of now'.format(arch))
+        else:
+            state_dict = load_state_dict_from_url(model_url, progress=progress)
+            model.load_state_dict(state_dict)
+
+    return model

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/segmentation/simple_seg_head.py

@@ -0,0 +1,63 @@
+# Simplified from MichaelFan01/STDC-Seg
+import torch.nn as nn
+from torch.nn import BatchNorm2d
+
+from ..builder import MODELS
+
+
+class ConvBNReLU(nn.Module):
+    def __init__(self, in_chan, out_chan, ks=3, stride=1, padding=1):
+        super(ConvBNReLU, self).__init__()
+        self.conv = nn.Conv2d(in_chan,
+                              out_chan,
+                              kernel_size = ks,
+                              stride = stride,
+                              padding = padding,
+                              bias = False)
+        self.bn = BatchNorm2d(out_chan)
+        self.relu = nn.ReLU()
+        self.init_weight()
+
+    def forward(self, x):
+        x = self.conv(x)
+        x = self.bn(x)
+        x = self.relu(x)
+        return x
+
+    def init_weight(self):
+        for ly in self.children():
+            if isinstance(ly, nn.Conv2d):
+                nn.init.kaiming_normal_(ly.weight, a=1)
+                if not ly.bias is None: nn.init.constant_(ly.bias, 0)
+
+
+@MODELS.register()
+class SimpleSegHead(nn.Module):
+    # Seg head (Generalized FCN head without dropout)
+    def __init__(self, in_channels, mid_channels, num_classes):
+        super().__init__()
+        self.conv = ConvBNReLU(in_channels, mid_channels, ks=3, stride=1, padding=1)
+        self.conv_out = nn.Conv2d(mid_channels, num_classes, kernel_size=1, bias=False)
+        self.init_weight()
+
+    def forward(self, x):
+        x = self.conv(x)
+        x = self.conv_out(x)
+        return x
+
+    def init_weight(self):
+        for ly in self.children():
+            if isinstance(ly, nn.Conv2d):
+                nn.init.kaiming_normal_(ly.weight, a=1)
+                if not ly.bias is None: nn.init.constant_(ly.bias, 0)
+
+    def get_params(self):
+        wd_params, nowd_params = [], []
+        for _, module in self.named_modules():
+            if isinstance(module, (nn.Linear, nn.Conv2d)):
+                wd_params.append(module.weight)
+                if not module.bias is None:
+                    nowd_params.append(module.bias)
+            elif isinstance(module, BatchNorm2d):
+                nowd_params += list(module.parameters())
+        return wd_params, nowd_params

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/swin.py

@@ -0,0 +1,626 @@
+# --------------------------------------------------------
+# Swin Transformer
+# Copyright (c) 2021 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+# Written by Ze Liu, Yutong Lin, Yixuan Wei
+# --------------------------------------------------------
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.utils.checkpoint as checkpoint
+import numpy as np
+from timm.models.layers import DropPath, to_2tuple, trunc_normal_
+from .builder import MODELS
+
+
+class Mlp(nn.Module):
+    """ Multilayer perceptron."""
+
+    def __init__(self, in_features, hidden_features=None, out_features=None, act_fn=F.gelu, drop=0.):
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        self.fc1 = nn.Linear(in_features, hidden_features)
+        self.act = act_fn
+        self.fc2 = nn.Linear(hidden_features, out_features)
+        self.drop = nn.Dropout(drop)
+
+    def forward(self, x):
+        x = self.fc1(x)
+        x = self.act(x)
+        x = self.drop(x)
+        x = self.fc2(x)
+        x = self.drop(x)
+        return x
+
+
+def window_partition(x, window_size):
+    """
+    Args:
+        x: (B, H, W, C)
+        window_size (int): window size
+
+    Returns:
+        windows: (num_windows*B, window_size, window_size, C)
+    """
+    B, H, W, C = x.shape
+    x = x.view(B, H // window_size, window_size, W // window_size, window_size, C)
+    windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C)
+    return windows
+
+
+def window_reverse(windows, window_size, H, W):
+    """
+    Args:
+        windows: (num_windows*B, window_size, window_size, C)
+        window_size (int): Window size
+        H (int): Height of image
+        W (int): Width of image
+
+    Returns:
+        x: (B, H, W, C)
+    """
+    B = int(windows.shape[0] / (H * W / float(window_size) / window_size))
+    x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1)
+    x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1)
+    return x
+
+
+class WindowAttention(nn.Module):
+    """ Window based multi-head self attention (W-MSA) module with relative position bias.
+    It supports both of shifted and non-shifted window.
+
+    Args:
+        dim (int): Number of input channels.
+        window_size (tuple[int]): The height and width of the window.
+        num_heads (int): Number of attention heads.
+        qkv_bias (bool, optional):  If True, add a learnable bias to query, key, value. Default: True
+        qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set
+        attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0
+        proj_drop (float, optional): Dropout ratio of output. Default: 0.0
+    """
+
+    def __init__(self, dim, window_size, num_heads, qkv_bias=True, qk_scale=None, attn_drop=0., proj_drop=0.):
+
+        super().__init__()
+        self.dim = dim
+        self.window_size = window_size  # Wh, Ww
+        self.num_heads = num_heads
+        head_dim = dim // num_heads
+        self.scale = qk_scale or head_dim ** -0.5
+
+        # define a parameter table of relative position bias
+        self.relative_position_bias_table = nn.Parameter(
+            torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), num_heads))  # 2*Wh-1 * 2*Ww-1, nH
+
+        # get pair-wise relative position index for each token inside the window
+        coords_h = torch.arange(self.window_size[0])
+        coords_w = torch.arange(self.window_size[1])
+        coords = torch.stack(torch.meshgrid([coords_h, coords_w]))  # 2, Wh, Ww
+        coords_flatten = torch.flatten(coords, 1)  # 2, Wh*Ww
+        relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :]  # 2, Wh*Ww, Wh*Ww
+        relative_coords = relative_coords.permute(1, 2, 0).contiguous()  # Wh*Ww, Wh*Ww, 2
+        relative_coords[:, :, 0] += self.window_size[0] - 1  # shift to start from 0
+        relative_coords[:, :, 1] += self.window_size[1] - 1
+        relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1
+        relative_position_index = relative_coords.sum(-1)  # Wh*Ww, Wh*Ww
+        self.register_buffer("relative_position_index", relative_position_index)
+
+        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
+        self.attn_drop = nn.Dropout(attn_drop)
+        self.proj = nn.Linear(dim, dim)
+        self.proj_drop = nn.Dropout(proj_drop)
+
+        trunc_normal_(self.relative_position_bias_table, std=.02)
+        self.softmax = nn.Softmax(dim=-1)
+
+    def forward(self, x, mask=None):
+        """ Forward function.
+
+        Args:
+            x: input features with shape of (num_windows*B, N, C)
+            mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None
+        """
+        B_, N, C = x.shape
+        qkv = self.qkv(x).reshape(B_, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
+        q, k, v = qkv[0], qkv[1], qkv[2]  # make torchscript happy (cannot use tensor as tuple)
+
+        q = q * self.scale
+        attn = (q @ k.transpose(-2, -1))
+
+        relative_position_bias = self.relative_position_bias_table[self.relative_position_index.view(-1)].view(
+            self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1)  # Wh*Ww,Wh*Ww,nH
+        relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous()  # nH, Wh*Ww, Wh*Ww
+        attn = attn + relative_position_bias.unsqueeze(0)
+
+        if mask is not None:
+            nW = mask.shape[0]
+            attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0)
+            attn = attn.view(-1, self.num_heads, N, N)
+            attn = self.softmax(attn)
+        else:
+            attn = self.softmax(attn)
+
+        attn = self.attn_drop(attn)
+
+        x = (attn @ v).transpose(1, 2).reshape(B_, N, C)
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x
+
+
+class SwinTransformerBlock(nn.Module):
+    """ Swin Transformer Block.
+
+    Args:
+        dim (int): Number of input channels.
+        num_heads (int): Number of attention heads.
+        window_size (int): Window size.
+        shift_size (int): Shift size for SW-MSA.
+        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
+        qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True
+        qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set.
+        drop (float, optional): Dropout rate. Default: 0.0
+        attn_drop (float, optional): Attention dropout rate. Default: 0.0
+        drop_path (float, optional): Stochastic depth rate. Default: 0.0
+        act_fn (nn.Module, optional): Activation layer. Default: F.gelu
+        norm_layer (nn.Module, optional): Normalization layer.  Default: nn.LayerNorm
+    """
+
+    def __init__(self, dim, num_heads, window_size=7, shift_size=0,
+                 mlp_ratio=4., qkv_bias=True, qk_scale=None, drop=0., attn_drop=0., drop_path=0.,
+                 act_fn=F.gelu, norm_layer=nn.LayerNorm):
+        super().__init__()
+        self.dim = dim
+        self.num_heads = num_heads
+        self.window_size = window_size
+        self.shift_size = shift_size
+        self.mlp_ratio = mlp_ratio
+        assert 0 <= self.shift_size < self.window_size, "shift_size must in 0-window_size"
+
+        self.norm1 = norm_layer(dim)
+        self.attn = WindowAttention(
+            dim, window_size=to_2tuple(self.window_size), num_heads=num_heads,
+            qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop)
+
+        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
+        self.norm2 = norm_layer(dim)
+        mlp_hidden_dim = int(dim * mlp_ratio)
+        self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_fn=act_fn, drop=drop)
+
+        self.H = None
+        self.W = None
+
+    def forward(self, x, mask_matrix):
+        """ Forward function.
+
+        Args:
+            x: Input feature, tensor size (B, H*W, C).
+            H, W: Spatial resolution of the input feature.
+            mask_matrix: Attention mask for cyclic shift.
+        """
+        B, L, C = x.shape
+        H, W = self.H, self.W
+        assert L == H * W, "input feature has wrong size"
+
+        shortcut = x
+        x = self.norm1(x)
+        x = x.view(B, H, W, C)
+
+        # pad feature maps to multiples of window size
+        pad_l = pad_t = 0
+        pad_r = (self.window_size - W % self.window_size) % self.window_size
+        pad_b = (self.window_size - H % self.window_size) % self.window_size
+        x = F.pad(x, (0, 0, pad_l, pad_r, pad_t, pad_b))
+        _, Hp, Wp, _ = x.shape
+
+        # cyclic shift
+        if self.shift_size > 0:
+            shifted_x = torch.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2))
+            attn_mask = mask_matrix
+        else:
+            shifted_x = x
+            attn_mask = None
+
+        # partition windows
+        x_windows = window_partition(shifted_x, self.window_size)  # nW*B, window_size, window_size, C
+        x_windows = x_windows.view(-1, self.window_size * self.window_size, C)  # nW*B, window_size*window_size, C
+
+        # W-MSA/SW-MSA
+        attn_windows = self.attn(x_windows, mask=attn_mask)  # nW*B, window_size*window_size, C
+
+        # merge windows
+        attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C)
+        shifted_x = window_reverse(attn_windows, self.window_size, Hp, Wp)  # B H' W' C
+
+        # reverse cyclic shift
+        if self.shift_size > 0:
+            x = torch.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2))
+        else:
+            x = shifted_x
+
+        if pad_r > 0 or pad_b > 0:
+            x = x[:, :H, :W, :].contiguous()
+
+        x = x.view(B, H * W, C)
+
+        # FFN
+        x = shortcut + self.drop_path(x)
+        x = x + self.drop_path(self.mlp(self.norm2(x)))
+
+        return x
+
+
+class PatchMerging(nn.Module):
+    """ Patch Merging Layer
+
+    Args:
+        dim (int): Number of input channels.
+        norm_layer (nn.Module, optional): Normalization layer.  Default: nn.LayerNorm
+    """
+    def __init__(self, dim, norm_layer=nn.LayerNorm):
+        super().__init__()
+        self.dim = dim
+        self.reduction = nn.Linear(4 * dim, 2 * dim, bias=False)
+        self.norm = norm_layer(4 * dim)
+
+    def forward(self, x, H, W):
+        """ Forward function.
+
+        Args:
+            x: Input feature, tensor size (B, H*W, C).
+            H, W: Spatial resolution of the input feature.
+        """
+        B, L, C = x.shape
+        assert L == H * W, "input feature has wrong size"
+
+        x = x.view(B, H, W, C)
+
+        # padding
+        pad_input = (H % 2 == 1) or (W % 2 == 1)
+        if pad_input:
+            x = F.pad(x, (0, 0, 0, W % 2, 0, H % 2))
+
+        x0 = x[:, 0::2, 0::2, :]  # B H/2 W/2 C
+        x1 = x[:, 1::2, 0::2, :]  # B H/2 W/2 C
+        x2 = x[:, 0::2, 1::2, :]  # B H/2 W/2 C
+        x3 = x[:, 1::2, 1::2, :]  # B H/2 W/2 C
+        x = torch.cat([x0, x1, x2, x3], -1)  # B H/2 W/2 4*C
+        x = x.view(B, -1, 4 * C)  # B H/2*W/2 4*C
+
+        x = self.norm(x)
+        x = self.reduction(x)
+
+        return x
+
+
+class BasicLayer(nn.Module):
+    """ A basic Swin Transformer layer for one stage.
+
+    Args:
+        dim (int): Number of feature channels
+        depth (int): Depths of this stage.
+        num_heads (int): Number of attention head.
+        window_size (int): Local window size. Default: 7.
+        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4.
+        qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True
+        qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set.
+        drop (float, optional): Dropout rate. Default: 0.0
+        attn_drop (float, optional): Attention dropout rate. Default: 0.0
+        drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0
+        norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm
+        downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None
+        use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False.
+    """
+
+    def __init__(self,
+                 dim,
+                 depth,
+                 num_heads,
+                 window_size=7,
+                 mlp_ratio=4.,
+                 qkv_bias=True,
+                 qk_scale=None,
+                 drop=0.,
+                 attn_drop=0.,
+                 drop_path=0.,
+                 norm_layer=nn.LayerNorm,
+                 downsample=None,
+                 use_checkpoint=False):
+        super().__init__()
+        self.window_size = window_size
+        self.shift_size = window_size // 2
+        self.depth = depth
+        self.use_checkpoint = use_checkpoint
+
+        # build blocks
+        self.blocks = nn.ModuleList([
+            SwinTransformerBlock(
+                dim=dim,
+                num_heads=num_heads,
+                window_size=window_size,
+                shift_size=0 if (i % 2 == 0) else window_size // 2,
+                mlp_ratio=mlp_ratio,
+                qkv_bias=qkv_bias,
+                qk_scale=qk_scale,
+                drop=drop,
+                attn_drop=attn_drop,
+                drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path,
+                norm_layer=norm_layer)
+            for i in range(depth)])
+
+        # patch merging layer
+        if downsample is not None:
+            self.downsample = downsample(dim=dim, norm_layer=norm_layer)
+        else:
+            self.downsample = None
+
+    def forward(self, x, H, W):
+        """ Forward function.
+
+        Args:
+            x: Input feature, tensor size (B, H*W, C).
+            H, W: Spatial resolution of the input feature.
+        """
+
+        # calculate attention mask for SW-MSA
+        Hp = int(np.ceil(float(H) / float(self.window_size))) * self.window_size
+        Wp = int(np.ceil(float(W) / float(self.window_size))) * self.window_size
+        img_mask = torch.zeros((1, Hp, Wp, 1), device=x.device)  # 1 Hp Wp 1
+        h_slices = (slice(0, -self.window_size),
+                    slice(-self.window_size, -self.shift_size),
+                    slice(-self.shift_size, None))
+        w_slices = (slice(0, -self.window_size),
+                    slice(-self.window_size, -self.shift_size),
+                    slice(-self.shift_size, None))
+        cnt = 0
+        for h in h_slices:
+            for w in w_slices:
+                img_mask[:, h, w, :] = cnt
+                cnt += 1
+
+        mask_windows = window_partition(img_mask, self.window_size)  # nW, window_size, window_size, 1
+        mask_windows = mask_windows.view(-1, self.window_size * self.window_size)
+        attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2)
+        attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill(attn_mask == 0, float(0.0))
+
+        for blk in self.blocks:
+            blk.H, blk.W = H, W
+            if self.use_checkpoint:
+                x = checkpoint.checkpoint(blk, x, attn_mask)
+            else:
+                x = blk(x, attn_mask)
+        if self.downsample is not None:
+            x_down = self.downsample(x, H, W)
+            Wh, Ww = (H + 1) // 2, (W + 1) // 2
+            return x, H, W, x_down, Wh, Ww
+        else:
+            return x, H, W, x, H, W
+
+
+class PatchEmbed(nn.Module):
+    """ Image to Patch Embedding
+
+    Args:
+        patch_size (int): Patch token size. Default: 4.
+        in_chans (int): Number of input image channels. Default: 3.
+        embed_dim (int): Number of linear projection output channels. Default: 96.
+        norm_layer (nn.Module, optional): Normalization layer. Default: None
+    """
+
+    def __init__(self, patch_size=4, in_chans=3, embed_dim=96, norm_layer=None):
+        super().__init__()
+        patch_size = to_2tuple(patch_size)
+        self.patch_size = patch_size
+
+        self.in_chans = in_chans
+        self.embed_dim = embed_dim
+
+        self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)
+        if norm_layer is not None:
+            self.norm = norm_layer(embed_dim)
+        else:
+            self.norm = None
+
+    def forward(self, x):
+        """Forward function."""
+        # padding
+        _, _, H, W = x.size()
+        if W % self.patch_size[1] != 0:
+            x = F.pad(x, (0, self.patch_size[1] - W % self.patch_size[1]))
+        if H % self.patch_size[0] != 0:
+            x = F.pad(x, (0, 0, 0, self.patch_size[0] - H % self.patch_size[0]))
+
+        x = self.proj(x)  # B C Wh Ww
+        if self.norm is not None:
+            Wh, Ww = x.size(2), x.size(3)
+            x = x.flatten(2).transpose(1, 2)
+            x = self.norm(x)
+            x = x.transpose(1, 2).view(-1, self.embed_dim, Wh, Ww)
+
+        return x
+
+
+@MODELS.register()
+class SwinTransformer(nn.Module):
+    """ Swin Transformer backbone.
+        A PyTorch impl of : `Swin Transformer: Hierarchical Vision Transformer using Shifted Windows`  -
+          https://arxiv.org/pdf/2103.14030
+
+    Args:
+        pretrain_img_size (int): Input image size for training the pretrained model,
+            used in absolute postion embedding. Default 224.
+        patch_size (int | tuple(int)): Patch size. Default: 4.
+        in_chans (int): Number of input image channels. Default: 3.
+        embed_dim (int): Number of linear projection output channels. Default: 96.
+        depths (tuple[int]): Depths of each Swin Transformer stage.
+        num_heads (tuple[int]): Number of attention head of each stage.
+        window_size (int): Window size. Default: 7.
+        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4.
+        qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True
+        qk_scale (float): Override default qk scale of head_dim ** -0.5 if set.
+        drop_rate (float): Dropout rate.
+        attn_drop_rate (float): Attention dropout rate. Default: 0.
+        drop_path_rate (float): Stochastic depth rate. Default: 0.2.
+        norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm.
+        ape (bool): If True, add absolute position embedding to the patch embedding. Default: False.
+        patch_norm (bool): If True, add normalization after patch embedding. Default: True.
+        out_indices (Sequence[int]): Output from which stages.
+        frozen_stages (int): Stages to be frozen (stop grad and set eval mode).
+            -1 means not freezing any parameters.
+        use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False.
+    """
+
+    def __init__(self,
+                 pretrain_img_size=224,
+                 patch_size=4,
+                 in_chans=3,
+                 embed_dim=96,
+                 depths=[2, 2, 6, 2],
+                 num_heads=[3, 6, 12, 24],
+                 window_size=7,
+                 mlp_ratio=4.,
+                 qkv_bias=True,
+                 qk_scale=None,
+                 drop_rate=0.,
+                 attn_drop_rate=0.,
+                 drop_path_rate=0.2,
+                 norm_layer=nn.LayerNorm,
+                 ape=False,
+                 patch_norm=True,
+                 out_indices=(0, 1, 2, 3),
+                 frozen_stages=-1,
+                 use_checkpoint=False,
+                 pretrained=None,
+                 chosen_stages=-1):
+        super().__init__()
+
+        self.pretrain_img_size = pretrain_img_size
+        self.num_layers = len(depths)
+        self.embed_dim = embed_dim
+        self.ape = ape
+        self.patch_norm = patch_norm
+        self.out_indices = out_indices
+        self.frozen_stages = frozen_stages
+        self.chosen_stages = chosen_stages
+
+        # split image into non-overlapping patches
+        self.patch_embed = PatchEmbed(
+            patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim,
+            norm_layer=norm_layer if self.patch_norm else None)
+
+        # absolute position embedding
+        if self.ape:
+            pretrain_img_size = to_2tuple(pretrain_img_size)
+            patch_size = to_2tuple(patch_size)
+            patches_resolution = [pretrain_img_size[0] // patch_size[0], pretrain_img_size[1] // patch_size[1]]
+
+            self.absolute_pos_embed = nn.Parameter(torch.zeros(1, embed_dim, patches_resolution[0], patches_resolution[1]))
+            trunc_normal_(self.absolute_pos_embed, std=.02)
+
+        self.pos_drop = nn.Dropout(p=drop_rate)
+
+        # stochastic depth
+        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))]  # stochastic depth decay rule
+
+        # build layers
+        self.layers = nn.ModuleList()
+        for i_layer in range(self.num_layers):
+            layer = BasicLayer(
+                dim=int(embed_dim * 2 ** i_layer),
+                depth=depths[i_layer],
+                num_heads=num_heads[i_layer],
+                window_size=window_size,
+                mlp_ratio=mlp_ratio,
+                qkv_bias=qkv_bias,
+                qk_scale=qk_scale,
+                drop=drop_rate,
+                attn_drop=attn_drop_rate,
+                drop_path=dpr[sum(depths[:i_layer]):sum(depths[:i_layer + 1])],
+                norm_layer=norm_layer,
+                downsample=PatchMerging if (i_layer < self.num_layers - 1) else None,
+                use_checkpoint=use_checkpoint)
+            self.layers.append(layer)
+
+        num_features = [int(embed_dim * 2 ** i) for i in range(self.num_layers)]
+        self.num_features = num_features
+
+        # add a norm layer for each output
+        for i_layer in out_indices:
+            layer = norm_layer(num_features[i_layer])
+            layer_name = f'norm{i_layer}'
+            self.add_module(layer_name, layer)
+
+        # initialize
+        if pretrained is None:
+            self.weight_initialization()
+        else:
+            assert isinstance(pretrained, str), 'pretrained must be a str'
+            self.load_checkpoint(pretrained=pretrained)
+
+        self._freeze_stages()
+
+    def _freeze_stages(self):
+        if self.frozen_stages >= 0:
+            self.patch_embed.eval()
+            for param in self.patch_embed.parameters():
+                param.requires_grad = False
+
+        if self.frozen_stages >= 1 and self.ape:
+            self.absolute_pos_embed.requires_grad = False
+
+        if self.frozen_stages >= 2:
+            self.pos_drop.eval()
+            for i in range(0, self.frozen_stages - 1):
+                m = self.layers[i]
+                m.eval()
+                for param in m.parameters():
+                    param.requires_grad = False
+
+    def weight_initialization(self):
+        for m in self.modules():
+            if isinstance(m, nn.Linear):
+                trunc_normal_(m.weight, std=.02)
+                if isinstance(m, nn.Linear) and m.bias is not None:
+                    nn.init.constant_(m.bias, 0)
+            elif isinstance(m, nn.LayerNorm):
+                nn.init.constant_(m.bias, 0)
+                nn.init.constant_(m.weight, 1.0)
+
+    def load_checkpoint(self, pretrained, strict=False):
+        state_dict = torch.load(pretrained)['model']
+        self.load_state_dict(state_dict, strict=strict)
+
+    def forward(self, x):
+        """Forward function."""
+        x = self.patch_embed(x)
+        Wh, Ww = x.size(2), x.size(3)
+        if self.ape:
+            # interpolate the position embedding to the corresponding size
+            absolute_pos_embed = F.interpolate(self.absolute_pos_embed, size=(Wh, Ww), mode='bicubic')
+            x = (x + absolute_pos_embed).flatten(2).transpose(1, 2)  # B Wh*Ww C
+        else:
+            x = x.flatten(2).transpose(1, 2)
+        x = self.pos_drop(x)
+
+        outs = []
+        for i in range(self.num_layers):
+            layer = self.layers[i]
+            x_out, H, W, x, Wh, Ww = layer(x, Wh, Ww)
+
+            if i in self.out_indices:
+                norm_layer = getattr(self, f'norm{i}')
+                x_out = norm_layer(x_out)
+
+                out = x_out.view(-1, H, W, self.num_features[i]).permute(0, 3, 1, 2).contiguous()
+                outs.append(out)
+        if self.chosen_stages == -1:
+            return tuple(outs)
+        else:
+            return tuple(outs)[self.chosen_stages]
+
+    def train(self, mode=True):
+        """Convert the model into training mode while keep layers freezed."""
+        super(SwinTransformer, self).train(mode)
+        self._freeze_stages()

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/transformer/__init__.py

@@ -0,0 +1,2 @@
+from .position_encoding import build_position_encoding
+from .transformer import Transformer, build_transformer

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/transformer/position_encoding.py

@@ -0,0 +1,90 @@
+# Copied and modified from facebookresearch/detr and liuruijin17/LSTR
+# mainly got rid of special types and argparse
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+# Positional encodings for transformers
+
+import math
+import torch
+from torch import nn
+
+from ..builder import MODELS
+
+
+@MODELS.register()
+class PositionEmbeddingSine(nn.Module):
+    """
+    This is a more standard version of the position embedding, very similar to the one
+    used by the Attention is all you need paper, generalized to work on images.
+    """
+    def __init__(self, num_pos_feats=64, temperature=10000, normalize=False, scale=None):
+        super().__init__()
+        self.num_pos_feats = num_pos_feats
+        self.temperature = temperature
+        self.normalize = normalize
+        if scale is not None and normalize is False:
+            raise ValueError("normalize should be True if scale is passed")
+        if scale is None:
+            scale = 2 * math.pi
+        self.scale = scale
+
+    def forward(self, x, mask):
+        assert mask is not None
+        not_mask = ~mask
+        y_embed = not_mask.cumsum(1, dtype=torch.float32)
+        x_embed = not_mask.cumsum(2, dtype=torch.float32)
+        if self.normalize:
+            eps = 1e-6
+            y_embed = y_embed / (y_embed[:, -1:, :] + eps) * self.scale
+            x_embed = x_embed / (x_embed[:, :, -1:] + eps) * self.scale
+
+        dim_t = torch.arange(self.num_pos_feats, dtype=torch.float32, device=x.device)
+        dim_t = self.temperature ** (2 * (dim_t // 2) / self.num_pos_feats)
+
+        pos_x = x_embed[:, :, :, None] / dim_t
+        pos_y = y_embed[:, :, :, None] / dim_t
+        pos_x = torch.stack((pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4).flatten(3)
+        pos_y = torch.stack((pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4).flatten(3)
+        pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2)
+
+        return pos
+
+
+@MODELS.register()
+class PositionEmbeddingLearned(nn.Module):
+    """
+    Absolute pos embedding, learned.
+    """
+    def __init__(self, num_pos_feats=256):
+        super().__init__()
+        self.row_embed = nn.Embedding(50, num_pos_feats)
+        self.col_embed = nn.Embedding(50, num_pos_feats)
+        self.reset_parameters()
+
+    def reset_parameters(self):
+        nn.init.uniform_(self.row_embed.weight)
+        nn.init.uniform_(self.col_embed.weight)
+
+    def forward(self, x):
+        h, w = x.shape[-2:]
+        i = torch.arange(w, device=x.device)
+        j = torch.arange(h, device=x.device)
+        x_emb = self.col_embed(i)
+        y_emb = self.row_embed(j)
+        pos = torch.cat([
+            x_emb.unsqueeze(0).repeat(h, 1, 1),
+            y_emb.unsqueeze(1).repeat(1, w, 1),
+        ], dim=-1).permute(2, 0, 1).unsqueeze(0).repeat(x.shape[0], 1, 1, 1)
+
+        return pos
+
+
+def build_position_encoding(hidden_dim, position_embedding):
+    N_steps = hidden_dim // 2
+    if position_embedding in ('v2', 'sine'):
+        position_embedding = PositionEmbeddingSine(N_steps, normalize=True)
+    elif position_embedding in ('v3', 'learned'):
+        position_embedding = PositionEmbeddingLearned(N_steps)
+    else:
+        raise ValueError(f"not supported {position_embedding}")
+
+    return position_embedding

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/transformer/transformer.py

@@ -0,0 +1,312 @@
+# Copied and modified from facebookresearch/detr and liuruijin17/LSTR
+# mainly got rid of special types and argparse, added comments
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+"""
+DETR Transformer class.
+Copy-paste from torch.nn.Transformer with modifications:
+    * positional encodings are passed in MHattention
+    * extra LN at the end of encoder is removed
+    * decoder returns a stack of activations from all decoding layers
+"""
+import copy
+from typing import Optional
+
+import torch
+import torch.nn.functional as F
+from torch import nn, Tensor
+
+from ..builder import MODELS
+
+
+@MODELS.register()
+class Transformer(nn.Module):
+    def __init__(self, d_model=512, nhead=8, num_encoder_layers=6,
+                 num_decoder_layers=6, dim_feedforward=2048, dropout=0.1,
+                 activation="relu", normalize_before=False,
+                 return_intermediate_dec=False):
+        super().__init__()
+
+        encoder_layer = TransformerEncoderLayer(d_model, nhead, dim_feedforward,
+                                                dropout, activation, normalize_before)
+        encoder_norm = nn.LayerNorm(d_model) if normalize_before else None
+        self.encoder = TransformerEncoder(encoder_layer, num_encoder_layers, encoder_norm)
+
+        decoder_layer = TransformerDecoderLayer(d_model, nhead, dim_feedforward,
+                                                dropout, activation, normalize_before)
+        decoder_norm = nn.LayerNorm(d_model)
+        self.decoder = TransformerDecoder(decoder_layer, num_decoder_layers, decoder_norm,
+                                          return_intermediate=return_intermediate_dec)
+
+        self._reset_parameters()
+
+        self.d_model = d_model
+        self.nhead = nhead
+
+    def _reset_parameters(self):
+        for p in self.parameters():
+            if p.dim() > 1:
+                nn.init.xavier_uniform_(p)
+
+    def forward(self, src, mask, query_embed, pos_embed):
+        # flatten NxCxHxW to HWxNxC (a sequence of length HW)
+        bs, c, h, w = src.shape
+        src = src.flatten(2).permute(2, 0, 1)
+        pos_embed = pos_embed.flatten(2).permute(2, 0, 1)
+        query_embed = query_embed.unsqueeze(1).repeat(1, bs, 1)
+        if mask is not None:
+            mask = mask.flatten(1)
+
+        tgt = torch.zeros_like(query_embed)
+        memory = self.encoder(src, src_key_padding_mask=mask, pos=pos_embed)
+        hs = self.decoder(tgt, memory, memory_key_padding_mask=mask,
+                          pos=pos_embed, query_pos=query_embed)
+
+        return hs.transpose(1, 2), memory.permute(1, 2, 0).view(bs, c, h, w)
+
+
+class TransformerEncoder(nn.Module):
+    def __init__(self, encoder_layer, num_layers, norm=None):
+        super().__init__()
+        self.layers = _get_clones(encoder_layer, num_layers)
+        self.num_layers = num_layers
+        self.norm = norm
+
+    def forward(self, src,
+                mask: Optional[Tensor] = None,
+                src_key_padding_mask: Optional[Tensor] = None,
+                pos: Optional[Tensor] = None):
+        output = src
+
+        for layer in self.layers:
+            output = layer(output, src_mask=mask,
+                           src_key_padding_mask=src_key_padding_mask, pos=pos)
+
+        if self.norm is not None:
+            output = self.norm(output)
+
+        return output
+
+
+class TransformerDecoder(nn.Module):
+    # Intermediate results are stacked at dimension 0 for aux loss
+    def __init__(self, decoder_layer, num_layers, norm=None, return_intermediate=False):
+        super().__init__()
+        self.layers = _get_clones(decoder_layer, num_layers)
+        self.num_layers = num_layers
+        self.norm = norm
+        self.return_intermediate = return_intermediate
+
+    def forward(self, tgt, memory,
+                tgt_mask: Optional[Tensor] = None,
+                memory_mask: Optional[Tensor] = None,
+                tgt_key_padding_mask: Optional[Tensor] = None,
+                memory_key_padding_mask: Optional[Tensor] = None,
+                pos: Optional[Tensor] = None,
+                query_pos: Optional[Tensor] = None):
+        output = tgt
+
+        intermediate = []
+
+        for layer in self.layers:
+            output = layer(output, memory, tgt_mask=tgt_mask,
+                           memory_mask=memory_mask,
+                           tgt_key_padding_mask=tgt_key_padding_mask,
+                           memory_key_padding_mask=memory_key_padding_mask,
+                           pos=pos, query_pos=query_pos)
+            if self.return_intermediate:
+                intermediate.append(self.norm(output))
+
+        if self.norm is not None:
+            output = self.norm(output)
+            if self.return_intermediate:
+                intermediate.pop()
+                intermediate.append(output)
+
+        if self.return_intermediate:
+            return torch.stack(intermediate)
+
+        return output.unsqueeze(0)
+
+
+class TransformerEncoderLayer(nn.Module):
+    def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1,
+                 activation="relu", normalize_before=False):
+        super().__init__()
+        self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout)
+        # Implementation of Feedforward model
+        self.linear1 = nn.Linear(d_model, dim_feedforward)
+        self.dropout = nn.Dropout(dropout)
+        self.linear2 = nn.Linear(dim_feedforward, d_model)
+
+        self.norm1 = nn.LayerNorm(d_model)
+        self.norm2 = nn.LayerNorm(d_model)
+        self.dropout1 = nn.Dropout(dropout)
+        self.dropout2 = nn.Dropout(dropout)
+
+        self.activation = _get_activation_fn(activation)
+        self.normalize_before = normalize_before
+
+    def forward_post(self,
+                     src,
+                     src_mask: Optional[Tensor] = None,
+                     src_key_padding_mask: Optional[Tensor] = None,
+                     pos: Optional[Tensor] = None):
+        q = k = _with_pos_embed(src, pos)
+        src2 = self.self_attn(q, k, value=src, attn_mask=src_mask,
+                              key_padding_mask=src_key_padding_mask)[0]
+        src = src + self.dropout1(src2)
+        src = self.norm1(src)
+        src2 = self.linear2(self.dropout(self.activation(self.linear1(src))))
+        src = src + self.dropout2(src2)
+        src = self.norm2(src)
+
+        return src
+
+    def forward_pre(self, src,
+                    src_mask: Optional[Tensor] = None,
+                    src_key_padding_mask: Optional[Tensor] = None,
+                    pos: Optional[Tensor] = None):
+        src2 = self.norm1(src)
+        q = k = _with_pos_embed(src2, pos)
+        src2 = self.self_attn(q, k, value=src2, attn_mask=src_mask,
+                              key_padding_mask=src_key_padding_mask)[0]
+        src = src + self.dropout1(src2)
+        src2 = self.norm2(src)
+        src2 = self.linear2(self.dropout(self.activation(self.linear1(src2))))
+        src = src + self.dropout2(src2)
+
+        return src
+
+    def forward(self, src,
+                src_mask: Optional[Tensor] = None,
+                src_key_padding_mask: Optional[Tensor] = None,
+                pos: Optional[Tensor] = None):
+        if self.normalize_before:
+            return self.forward_pre(src, src_mask, src_key_padding_mask, pos)
+        else:
+            return self.forward_post(src, src_mask, src_key_padding_mask, pos)
+
+
+class TransformerDecoderLayer(nn.Module):
+    def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1,
+                 activation="relu", normalize_before=False):
+        super().__init__()
+        self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout)
+        self.multihead_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout)
+        # Implementation of feed-forward model
+        self.linear1 = nn.Linear(d_model, dim_feedforward)
+        self.dropout = nn.Dropout(dropout)
+        self.linear2 = nn.Linear(dim_feedforward, d_model)
+
+        self.norm1 = nn.LayerNorm(d_model)
+        self.norm2 = nn.LayerNorm(d_model)
+        self.norm3 = nn.LayerNorm(d_model)
+        self.dropout1 = nn.Dropout(dropout)
+        self.dropout2 = nn.Dropout(dropout)
+        self.dropout3 = nn.Dropout(dropout)
+
+        self.activation = _get_activation_fn(activation)
+        self.normalize_before = normalize_before
+
+    def forward_post(self, tgt, memory,
+                     tgt_mask: Optional[Tensor] = None,
+                     memory_mask: Optional[Tensor] = None,
+                     tgt_key_padding_mask: Optional[Tensor] = None,
+                     memory_key_padding_mask: Optional[Tensor] = None,
+                     pos: Optional[Tensor] = None,
+                     query_pos: Optional[Tensor] = None):
+        q = k = _with_pos_embed(tgt, query_pos)
+        tgt2 = self.self_attn(q, k, value=tgt, attn_mask=tgt_mask,
+                              key_padding_mask=tgt_key_padding_mask)[0]
+        tgt = tgt + self.dropout1(tgt2)
+        tgt = self.norm1(tgt)
+        tgt2 = self.multihead_attn(query=_with_pos_embed(tgt, query_pos),
+                                   key=_with_pos_embed(memory, pos),
+                                   value=memory, attn_mask=memory_mask,
+                                   key_padding_mask=memory_key_padding_mask)[0]
+        tgt = tgt + self.dropout2(tgt2)
+        tgt = self.norm2(tgt)
+        tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt))))
+        tgt = tgt + self.dropout3(tgt2)
+        tgt = self.norm3(tgt)
+
+        return tgt
+
+    def forward_pre(self, tgt, memory,
+                    tgt_mask: Optional[Tensor] = None,
+                    memory_mask: Optional[Tensor] = None,
+                    tgt_key_padding_mask: Optional[Tensor] = None,
+                    memory_key_padding_mask: Optional[Tensor] = None,
+                    pos: Optional[Tensor] = None,
+                    query_pos: Optional[Tensor] = None):
+        tgt2 = self.norm1(tgt)
+        q = k = _with_pos_embed(tgt2, query_pos)
+        tgt2 = self.self_attn(q, k, value=tgt2, attn_mask=tgt_mask,
+                              key_padding_mask=tgt_key_padding_mask)[0]
+        tgt = tgt + self.dropout1(tgt2)
+        tgt2 = self.norm2(tgt)
+        tgt2 = self.multihead_attn(query=_with_pos_embed(tgt2, query_pos),
+                                   key=_with_pos_embed(memory, pos),
+                                   value=memory, attn_mask=memory_mask,
+                                   key_padding_mask=memory_key_padding_mask)[0]
+        tgt = tgt + self.dropout2(tgt2)
+        tgt2 = self.norm3(tgt)
+        tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt2))))
+        tgt = tgt + self.dropout3(tgt2)
+
+        return tgt
+
+    def forward(self, tgt, memory,
+                tgt_mask: Optional[Tensor] = None,
+                memory_mask: Optional[Tensor] = None,
+                tgt_key_padding_mask: Optional[Tensor] = None,
+                memory_key_padding_mask: Optional[Tensor] = None,
+                pos: Optional[Tensor] = None,
+                query_pos: Optional[Tensor] = None):
+        if self.normalize_before:
+            return self.forward_pre(tgt, memory, tgt_mask, memory_mask,
+                                    tgt_key_padding_mask, memory_key_padding_mask, pos, query_pos)
+        else:
+            return self.forward_post(tgt, memory, tgt_mask, memory_mask,
+                                     tgt_key_padding_mask, memory_key_padding_mask, pos, query_pos)
+
+
+def _get_clones(module, N):
+    return nn.ModuleList([copy.deepcopy(module) for _ in range(N)])
+
+
+def build_transformer(hidden_dim,
+                      dropout,
+                      nheads,
+                      dim_feedforward,
+                      enc_layers,
+                      dec_layers,
+                      pre_norm=False,
+                      return_intermediate_dec=False):
+
+    return Transformer(
+        d_model=hidden_dim,
+        dropout=dropout,
+        nhead=nheads,
+        dim_feedforward=dim_feedforward,
+        num_encoder_layers=enc_layers,
+        num_decoder_layers=dec_layers,
+        normalize_before=pre_norm,
+        return_intermediate_dec=return_intermediate_dec,
+    )
+
+
+def _with_pos_embed(tensor, pos: Optional[Tensor]):
+    return tensor if pos is None else tensor + pos
+
+
+def _get_activation_fn(activation):
+    """Return an activation function given a string"""
+    if activation == "relu":
+        return F.relu
+    elif activation == "gelu":
+        return F.gelu
+    elif activation == "glu":
+        return F.glu
+    else:
+        raise RuntimeError(F"activation should be relu/gelu, not {activation}.")

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/utils.py

@@ -0,0 +1,4 @@
+try:
+    from torch.hub import load_state_dict_from_url
+except ImportError:
+    from torch.utils.model_zoo import load_url as load_state_dict_from_url

algorithms/lane_ufld/code/pytorch-auto-drive-master/utils/models/vgg_encoder.py

@@ -0,0 +1,32 @@
+from torch import nn as nn
+from torchvision.models import vgg16_bn
+
+from .builder import MODELS
+
+
+@MODELS.register()
+class VGG16(nn.Module):
+    # Modified VGG16 backbone in DeepLab-LargeFOV,
+    # note that due to legacy implementation issues,
+    # the converted fully-connected layers (FC-6 FC-7) are not included here.
+    # jcdubron/scnn_pytorch
+
+    def __init__(self, pretrained=True):
+        super().__init__()
+        self.pretrained = pretrained
+        self.net = vgg16_bn(pretrained=self.pretrained).features
+        for i in [34, 37, 40]:
+            conv = self.net._modules[str(i)]
+            dilated_conv = nn.Conv2d(
+                conv.in_channels, conv.out_channels, conv.kernel_size, stride=conv.stride,
+                padding=tuple(p * 2 for p in conv.padding), dilation=2, bias=(conv.bias is not None)
+            )
+            dilated_conv.load_state_dict(conv.state_dict())
+            self.net._modules[str(i)] = dilated_conv
+        self.net._modules.pop('33')
+        self.net._modules.pop('43')
+
+    def forward(self, x):
+        x = self.net(x)
+
+        return x