65 lines
3.2 KiB
Python
65 lines
3.2 KiB
Python
""" Publishes a ROS topic with name /depth/compressed and shows the image on OpenCV window.
|
|
Issues: rqt_image_view is not showing the image due to some data conversion issues but OpenCV is showing the image."""
|
|
import os
|
|
import hydra
|
|
import open3d as o3d
|
|
from digit_depth.third_party import geom_utils
|
|
from digit_depth.digit import DigitSensor
|
|
from digit_depth.train import MLP
|
|
from digit_depth.train.prepost_mlp import *
|
|
from attrdict import AttrDict
|
|
from digit_depth.third_party import vis_utils
|
|
seed = 42
|
|
torch.seed = seed
|
|
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
|
|
base_path = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
|
|
|
|
|
|
@hydra.main(config_path="/home/shuk/digit-depth/config", config_name="rgb_to_normal.yaml", version_base=None)
|
|
def show_point_cloud(cfg):
|
|
view_params = AttrDict({'fov': 60, 'front': [-0.1, 0.1, 0.1], 'lookat': [
|
|
-0.001, -0.01, 0.01], 'up': [0.04, -0.05, 0.190], 'zoom': 2.5})
|
|
vis3d = vis_utils.Visualizer3d(base_path=base_path, view_params=view_params)
|
|
|
|
# projection params
|
|
T_cam_offset = torch.tensor(cfg.sensor.T_cam_offset)
|
|
proj_mat = torch.tensor(cfg.sensor.P)
|
|
model = torch.load(cfg.model_path).to(device)
|
|
model.eval()
|
|
# base image depth map
|
|
base_img = cv2.imread(cfg.base_img_path)
|
|
base_img = preproc_mlp(base_img)
|
|
base_img_proc = model(base_img).cpu().detach().numpy()
|
|
base_img_proc, normal_base = post_proc_mlp(base_img_proc)
|
|
# get gradx and grady
|
|
gradx_base, grady_base = geom_utils._normal_to_grad_depth(img_normal=base_img_proc, gel_width=cfg.sensor.gel_width,
|
|
gel_height=cfg.sensor.gel_height, bg_mask=None)
|
|
|
|
# reconstruct depth
|
|
img_depth_base = geom_utils._integrate_grad_depth(gradx_base, grady_base, boundary=None, bg_mask=None,
|
|
max_depth=0.0237)
|
|
img_depth_base = img_depth_base.detach().cpu().numpy() # final depth image for base image
|
|
# setup digit sensor
|
|
digit = DigitSensor(30, "QVGA", cfg.sensor.serial_num)
|
|
digit_call = digit()
|
|
while True:
|
|
frame = digit_call.get_frame()
|
|
img_np = preproc_mlp(frame)
|
|
img_np = model(img_np).detach().cpu().numpy()
|
|
img_np, normal_img = post_proc_mlp(img_np)
|
|
# get gradx and grady
|
|
gradx_img, grady_img = geom_utils._normal_to_grad_depth(img_normal=img_np, gel_width=cfg.sensor.gel_width,
|
|
gel_height=cfg.sensor.gel_height,bg_mask=None)
|
|
# reconstruct depth
|
|
img_depth = geom_utils._integrate_grad_depth(gradx_img, grady_img, boundary=None, bg_mask=None, max_depth=0.0237)
|
|
view_mat = torch.eye(4) # torch.inverse(T_cam_offset)
|
|
# Project depth to 3D
|
|
points3d = geom_utils.depth_to_pts3d(depth=img_depth, P=proj_mat, V=view_mat, params=cfg.sensor)
|
|
points3d = geom_utils.remove_background_pts(points3d, bg_mask=None)
|
|
cloud = o3d.geometry.PointCloud()
|
|
clouds = geom_utils.init_points_to_clouds(clouds=[copy.deepcopy(cloud)], points3d=[points3d])
|
|
vis_utils.visualize_geometries_o3d(vis3d=vis3d, clouds=clouds)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
show_point_cloud() |