Upload 372 files

.gitattributes

@@ -36,3 +36,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 DAP-main-2/assets/depth_teaser2_00.png filter=lfs diff=lfs merge=lfs -text
 DAP-main-2/assets/depth_teaser2.pdf filter=lfs diff=lfs merge=lfs -text
 DAP-main-2/assets/teaser.jpg filter=lfs diff=lfs merge=lfs -text
+assets/depth_teaser2_00.png filter=lfs diff=lfs merge=lfs -text
+assets/depth_teaser2.pdf filter=lfs diff=lfs merge=lfs -text
+assets/teaser.jpg filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Insta360 Research Team
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,103 @@
+<h1 align="center">
+Depth Any Panoramas:<br>
+A Foundation Model for Panoramic Depth Estimation
+</h1>
+
+
+<p align="center">
+  <a href="https://linxin0.github.io"><b>Xin Lin</b></a> ·
+  <a href="#"><b>Meixi Song</b></a> ·
+  <a href="#"><b>Dizhe Zhang</b></a> ·
+  <a href="#"><b>Wenxuan Lu</b></a> ·
+  <a href="https://haodong2000.github.io"><b>Haodong Li</b></a>
+  <br>
+  <a href="#"><b>Bo Du</b></a> ·
+  <a href="#"><b>Ming-Hsuan Yang</b></a> ·
+  <a href="#"><b>Truong Nguyen</b></a> ·
+  <a href="http://luqi.info"><b>Lu Qi</b></a>
+</p>
+
+
+
+<p align="center">
+  <a href='https://arxiv.org/abs/2512.16913'><img src='https://img.shields.io/badge/arXiv-Paper-red?logo=arxiv&logoColor=white' alt='arXiv'></a>
+  <a href='https://insta360-research-team.github.io/DAP_website/'><img src='https://img.shields.io/badge/Project_Page-Website-green?logo=insta360&logoColor=white' alt='Project Page'></a>
+  <a href=''><img src='https://img.shields.io/badge/%F0%9F%93%88%20Hugging%20Face-Dataset-yellow'></a>
+  <a href='https://huggingface.co/spaces/Insta360-Research/DAP'><img src='https://img.shields.io/badge/🚀%20Hugging%20Face-Demo-orange'></a>
+</p>
+
+![teaser](assets/depth_teaser2_00.png)
+
+
+
+## 🔨 Installation
+
+Clone the repo first:
+
+```Bash
+git clone https://github.com/Insta360-Research-Team/DAP
+cd DAP
+```
+
+(Optional) Create a fresh conda env:
+
+```Bash
+conda create -n dap python=3.12
+conda activate dap
+```
+
+Install necessary packages (torch > 2):
+
+```Bash
+# pytorch (select correct CUDA version, we test our code on torch==2.7.1 and torchvision==0.22.1)
+pip install torch==2.7.1 torchvision==0.22.1
+
+# other dependencies
+pip install -r requirements.txt
+```
+
+## 🖼️ Dataset
+
+The training dataset will be open soon.
+
+
+## 🤝 Pre-trained model
+
+Please download the pretrained model: https://huggingface.co/Insta360-Research/DAP-weights
+
+
+## 📒 Inference
+
+```Bash
+python test/infer.py 
+```
+
+
+## 🚀 Evaluation
+
+
+```Bash
+python test/eval.py 
+```
+
+
+
+
+## 🤝 Acknowledgement
+
+We appreciate the open source of the following projects:
+
+* [PanDA](https://caozidong.github.io/PanDA_Depth/)
+* [Depth-Anything-V2](https://github.com/DepthAnything/Depth-Anything-V2)
+
+
+## Citation
+```
+@article{lin2025dap,
+          title={Depth Any Panoramas: A Foundation Model for Panoramic Depth Estimation},
+          author={Lin, Xin and Song, Meixi and Zhang, Dizhe and Lu, Wenxuan and Li, Haodong and Du, Bo and Yang, Ming-Hsuan and Nguyen, Truong and Qi, Lu},
+          journal={arXiv},
+          year={2025}
+        }
+```
+

assets/depth_teaser2.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28e8d4fecc3a5bd905ffead457d35f3602c98e78976c75e5da6317286ae4a385
+size 1872942

config/infer.yaml

@@ -0,0 +1,19 @@
+model:
+  name: dap
+  args:
+    midas_model_type: vitl         
+    fine_tune_type: hypersim       
+    min_depth: 0.01
+    max_depth: 1.0
+    train_decoder: True
+
+median_align: False
+load_weights_dir: /home/tione/notebook/home/songmeixi_insta360.com/depth/panda_orgindual/ckpt_save/1111/trainw1_2_dualw1_2/weights_0
+input:
+  height: 512
+  width: 1024
+inference:
+  batch_size: 1
+  num_workers: 1
+  save_colormap: True       
+  colormap_type: jet        

config/test.yaml

@@ -0,0 +1,71 @@
+test_dataset_1:
+  name: stanford2d3d
+  root_path: /home/tione/notebook/home/wenxuan/PanDA/data/stanford2d3d
+  list_path: datasets/stanford2d3d_test.txt
+  args:
+    height: 512
+    width: 1024
+    repeat: 1
+    augment_color: False
+    augment_flip: False
+    augment_rotation: False
+  batch_size: 32
+  num_workers: 64
+
+# test_dataset_1:
+#   name: insta23k
+#   root_path: /home/tione/notebook/nfs/MLUAV_Data
+#   list_path: datasets/instadata_list_test.txt
+#   args:
+#     height: 512
+#     width: 1024
+#     repeat: 1
+#     augment_color: False
+#     augment_flip: False
+#     augment_rotation: False
+#   batch_size: 32
+#   num_workers: 64
+
+# test_dataset_1:
+#   name: deep360
+#   root_path: /home/tione/notebook/home/wenxuan/PanDA/data/Deep360
+#   list_path: datasets/deep360_test_final.txt
+#   args:
+#     height: 512
+#     width: 1024
+#     repeat: 1
+#     augment_color: False
+#     augment_flip: False
+#     augment_rotation: False
+#   batch_size: 32
+#   num_workers: 24
+
+# test_dataset_1:
+#   name: m3d
+#   root_path: /home/tione/notebook/home/wenxuan/PanDA/data/M3D
+#   list_path: datasets/m3d_test.txt
+#   args:
+#     height: 512
+#     width: 1024
+#     repeat: 1
+#     augment_color: False
+#     augment_flip: False
+#     augment_rotation: False
+#   batch_size: 32
+#   num_workers: 24
+
+
+model:
+  name: dap
+  args:
+    midas_model_type: vitl
+    fine_tune_type: 
+    min_depth: 0.001
+    max_depth: 1.0             
+    train_decoder: True
+
+median_align: False
+
+
+
+load_weights_dir: /home/tione/notebook/home/songmeixi_insta360.com/depth/panda_orgindual/ckpt_save/1111/trainw1_2_dualw1_2/weights_0

datasets/M3D.py

@@ -0,0 +1,135 @@
+from __future__ import print_function
+import os
+import cv2
+import numpy as np
+import random
+import pyexr
+import torch
+from torch.utils import data
+from torchvision import transforms
+from torchvision.transforms import Compose
+
+from PIL import Image, ImageOps, ImageFilter
+import torch.nn.functional as F
+from einops import rearrange
+
+def read_list(list_file):
+    rgb_depth_list = []
+    with open(list_file) as f:
+        lines = f.readlines()
+        for line in lines:
+            rgb_depth_list.append(line.strip().split(" "))
+    return rgb_depth_list
+
+class M3D(data.Dataset):
+    """The M3D Dataset"""
+
+    def __init__(self, root_dir, list_file, height=504, width=1008, color_augmentation=True,
+                 LR_filp_augmentation=True, yaw_rotation_augmentation=True, repeat=1, is_training=False):
+        """
+        Args:
+            root_dir (string): Directory of the Stanford2D3D Dataset.
+            list_file (string): Path to the txt file contain the list of image and depth files.
+            height, width: input size.
+            disable_color_augmentation, disable_LR_filp_augmentation,
+            disable_yaw_rotation_augmentation: augmentation options.
+            is_training (bool): True if the dataset is the training set.
+        """
+        self.root_dir = root_dir
+
+        self.w = width
+        self.h = height
+
+        self.max_depth_meters = 100.0
+        self.min_depth_meters = 0.01
+
+        self.color_augmentation = color_augmentation
+        self.LR_filp_augmentation = LR_filp_augmentation
+        self.yaw_rotation_augmentation = yaw_rotation_augmentation
+
+        if self.color_augmentation:
+            try:
+                self.brightness = (0.8, 1.2)
+                self.contrast = (0.8, 1.2)
+                self.saturation = (0.8, 1.2)
+                self.hue = (-0.1, 0.1)
+                self.color_aug= transforms.ColorJitter(
+                    self.brightness, self.contrast, self.saturation, self.hue)
+            except TypeError:
+                self.brightness = 0.2
+                self.contrast = 0.2
+                self.saturation = 0.2
+                self.hue = 0.1
+                self.color_aug = transforms.ColorJitter(
+                    self.brightness, self.contrast, self.saturation, self.hue)
+
+        self.is_training = is_training
+
+        self.to_tensor = transforms.ToTensor()
+        self.normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+
+        self.rgb_depth_list = read_list(list_file)
+
+    def __len__(self):
+        return len(self.rgb_depth_list)
+
+    def __getitem__(self, idx):
+
+        # Read and process the image file
+        rgb_name = os.path.join(self.root_dir, self.rgb_depth_list[idx][0])
+        rgb = cv2.imread(rgb_name)
+        # cv2.imwrite('label_rgb.jpg', rgb)
+        rgb = cv2.cvtColor(rgb, cv2.COLOR_BGR2RGB)
+        rgb = cv2.resize(rgb, dsize=(self.w, self.h), interpolation=cv2.INTER_CUBIC)
+        
+        # Read and process the depth file
+        depth_name = os.path.join(self.root_dir, self.rgb_depth_list[idx][1])
+        # gt_depth = cv2.imread(depth_name, -1)
+        # gt_depth = cv2.resize(gt_depth, dsize=(self.w, self.h), interpolation=cv2.INTER_NEAREST)
+        # gt_depth = gt_depth.astype(float)/4000
+        # gt_depth[gt_depth > self.max_depth_meters+1] = self.max_depth_meters + 1
+
+        gt_depth = pyexr.open(depth_name).get()
+        gt_depth = gt_depth[:, :, 0]
+        gt_depth = cv2.resize(gt_depth, dsize=(self.w, self.h), interpolation=cv2.INTER_NEAREST)
+        gt_depth[gt_depth > self.max_depth_meters+1] = self.max_depth_meters + 1
+
+        if self.is_training and self.yaw_rotation_augmentation:
+            # random yaw rotation
+            roll_idx = random.randint(0, self.w)
+            rgb = np.roll(rgb, roll_idx, 1)
+            gt_depth = np.roll(gt_depth, roll_idx, 1)
+
+        if self.is_training and self.LR_filp_augmentation and random.random() > 0.5:
+            rgb = cv2.flip(rgb, 1)
+            gt_depth = cv2.flip(gt_depth, 1)
+
+        if self.is_training and self.color_augmentation and random.random() > 0.5:
+            aug_rgb = np.asarray(self.color_aug(transforms.ToPILImage()(rgb)))
+        else:
+            aug_rgb = rgb.copy()
+
+        aug_rgb = self.to_tensor(aug_rgb.copy())
+
+        gt_depth = torch.from_numpy(np.expand_dims(gt_depth, axis=0)).to(torch.float32)
+
+        val_mask = ((gt_depth > 0) & (gt_depth <= self.max_depth_meters)& ~torch.isnan(gt_depth))
+
+        # _min, _max = torch.quantile(gt_depth[val_mask], torch.tensor([0.02, 1 - 0.02]),)
+        # gt_depth = gt_depth / 2560.0
+        gt_depth_norm = gt_depth / 100.0
+        gt_depth_norm = torch.clip(gt_depth_norm, 0.001, 1.0)
+
+        # print(gt_depth_norm.shape)
+        # Conduct output
+        inputs = {}
+
+        inputs["rgb"] = self.normalize(aug_rgb)
+        inputs["gt_depth"] = gt_depth_norm 
+        inputs["val_mask"] = val_mask # 合法区域，不是全true，真把不能用的��域划出来了；其他参与训练的数据集是全true的（除了投影数据集）
+        inputs["mask_100"] = (gt_depth > 0) & (gt_depth <= 100)
+        # 对于这个数据集，mask_100设定为全true的，因为求不出来。大于100米的深度gt也有可能是玻璃镜子等物体，反正这个数据集也不参加训练
+        
+        # 这个数据集中，模型预测的mask100应该是被val_mask涵盖的，所以mask100理论上没有影响
+        # val_mask控制计算指标的区域
+        return inputs

datasets/__init__.py

@@ -0,0 +1,4 @@
+from .stanford2d3d import Stanford2D3D
+from .deep360 import Deep360
+from .insta23k import Insta23k
+from .M3D import M3D