Skip to content

Commit

Permalink
Merge pull request #164 from bacetiner/master
Browse files Browse the repository at this point in the history 
Updated FacadeParser to depthmap-based height prediction and added keyless satellite and street-level image downloading
  • Loading branch information
@bacetiner
bacetiner authored Dec 29, 2023
2 parents 7297305 + 114d5df commit b349f54
Show file tree
Hide file tree
Showing 3 changed files with 579 additions and 136 deletions.
73 changes: 29 additions & 44 deletions brails/InventoryGenerator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
# Satish Rao
#
# Last updated:
# 05-06-2023
# 12-29-2023

import random
import sys
Expand Down Expand Up @@ -265,7 +265,7 @@ def write_to_dataframe(df,predictions,column,imtype='street_images'):
'garageExists')
self.inventory['garageExists'] = self.inventory['garageExists'].astype(dtype="boolean")

elif attribute=='numstories' and 'storyModel' not in locals():
elif attribute=='numstories':
# Initialize the floor detector object:
storyModel = NFloorDetector()

Expand Down Expand Up @@ -325,28 +325,13 @@ def write_to_dataframe(df,predictions,column,imtype='street_images'):
'satellite_images')

elif attribute in ['buildingheight','roofeaveheight','roofpitch']:
if 'storyModel' not in locals():
# Initialize the floor detector object:
storyModel = NFloorDetector()

# Call the floor detector to determine number of floors of
# buildings in each image:
storyModel.predict(imstreet)

# Write the results to the inventory DataFrame:
self.inventory = write_to_dataframe(self.inventory,
[storyModel.system_dict['infer']['images'],
storyModel.system_dict['infer']['predictions']],
'NumberOfStories')
self.inventory['NumberOfStories'] = self.inventory['NumberOfStories'].astype(dtype='Int64')

if 'facadeParserModel' not in locals():
if 'facadeParserModel' not in locals():
# Initialize the facade parser object:
facadeParserModel = FacadeParser()

# Call the facade parser to determine the requested
# attribute for each building:
facadeParserModel.predict(image_handler,storyModel)
facadeParserModel.predict(image_handler)

self.inventory = write_to_dataframe(self.inventory,
[facadeParserModel.predictions['image'].to_list(),
Expand Down Expand Up @@ -383,35 +368,35 @@ def write_to_dataframe(df,predictions,column,imtype='street_images'):
# Merge the DataFrame of predicted attributes with the DataFrame of
# incomplete inventory and print the resulting table to the output file
# titled IncompleteInventory.csv:
dfout2merge['fp_as_string'] = dfout2merge['Footprint'].apply(lambda x: "".join(str(x)))
# dfout2merge['fp_as_string'] = dfout2merge['Footprint'].apply(lambda x: "".join(str(x)))

dfout_incomp = self.incompleteInventory.copy(deep=True)
dfout_incomp['fp_as_string'] = dfout_incomp['Footprint'].apply(lambda x: "".join(str(x)))
# dfout_incomp = self.incompleteInventory.copy(deep=True)
# dfout_incomp['fp_as_string'] = dfout_incomp['Footprint'].apply(lambda x: "".join(str(x)))

dfout_incomp = pd.merge(left=dfout_incomp,
right=dfout2merge.drop(columns=['Footprint'], errors='ignore'),
how='left', left_on=['fp_as_string','PlanArea'],
right_on=['fp_as_string','PlanArea'],
sort=False)
# dfout_incomp = pd.merge(left=dfout_incomp,
# right=dfout2merge.drop(columns=['Footprint'], errors='ignore'),
# how='left', left_on=['fp_as_string','PlanArea'],
# right_on=['fp_as_string','PlanArea'],
# sort=False)

dfout_incomp = dfout2merge.append(dfout_incomp[dfout_incomp.roofshape.isnull()])
dfout_incomp = dfout_incomp.reset_index(drop=True).drop(columns=['fp_as_string'], errors='ignore')
# dfout_incomp = dfout2merge.append(dfout_incomp[dfout_incomp.roofshape.isnull()])
# dfout_incomp = dfout_incomp.reset_index(drop=True).drop(columns=['fp_as_string'], errors='ignore')

self.incompleteInventory = dfout_incomp.copy(deep=True)
# self.incompleteInventory = dfout_incomp.copy(deep=True)

dfout_incomp4print = dfout_incomp.copy(deep=True)
for index, row in dfout_incomp.iterrows():
dfout_incomp4print.loc[index, 'Footprint'] = ('{"type":"Feature","geometry":' +
'{"type":"Polygon","coordinates":[' +
f"""{row['Footprint']}""" +
']},"properties":{}}')
centroid = Polygon(row['Footprint']).centroid
dfout_incomp4print.loc[index, 'Latitude'] = centroid.y
dfout_incomp4print.loc[index, 'Longitude'] = centroid.x
# dfout_incomp4print = dfout_incomp.copy(deep=True)
# for index, row in dfout_incomp.iterrows():
# dfout_incomp4print.loc[index, 'Footprint'] = ('{"type":"Feature","geometry":' +
# '{"type":"Polygon","coordinates":[' +
# f"""{row['Footprint']}""" +
# ']},"properties":{}}')
# centroid = Polygon(row['Footprint']).centroid
# dfout_incomp4print.loc[index, 'Latitude'] = centroid.y
# dfout_incomp4print.loc[index, 'Longitude'] = centroid.x

cols = [col for col in dfout_incomp4print.columns if col!='Footprint']
new_cols = ['Latitude','Longitude'] + cols[:-2] + ['Footprint']
dfout_incomp4print = dfout_incomp4print[new_cols]
# cols = [col for col in dfout_incomp4print.columns if col!='Footprint']
# new_cols = ['Latitude','Longitude'] + cols[:-2] + ['Footprint']
# dfout_incomp4print = dfout_incomp4print[new_cols]

dfout_incomp4print.to_csv('IncompleteInventory.csv', index=True, index_label='id', na_rep='NA')
print('Incomplete inventory data available in IncompleteInventory.csv')
# dfout_incomp4print.to_csv('IncompleteInventory.csv', index=True, index_label='id', na_rep='NA')
# print('Incomplete inventory data available in IncompleteInventory.csv')
116 changes: 82 additions & 34 deletions brails/modules/FacadeParser/FacadeParser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
# Barbaros Cetiner
#
# Last updated:
# 05-19-2022
# 12-29-2023

import math
import torch
Expand All @@ -49,22 +49,25 @@
import torchvision.transforms as T
from shapely.geometry import Polygon
from tqdm import tqdm
from random import random
from sklearn.linear_model import LinearRegression
from copy import deepcopy

class FacadeParser:

def __init__(self):
self.cam_elevs = []
self.depthmaps = []
self.footprints = []
self.streetScales = []
self.model_path = []
self.street_images = []
self.model_path = []

def predict(self,imhandler,storymodel,model_path='tmp/models/facadeParser.pth',
def predict(self,imhandler,model_path='tmp/models/facadeParser.pth',
save_segimages=False):
self.cam_elevs = imhandler.cam_elevs[:]
self.depthmaps = imhandler.depthmaps[:]
self.footprints = imhandler.footprints[:]
self.streetScales = imhandler.streetScales[:]
self.street_images = imhandler.street_images[:]
self.model_path = model_path
self.street_images = imhandler.street_images[:]

def compute_roofrun(footprint):
# Find the mimumum area rectangle that fits around the footprint
Expand Down Expand Up @@ -269,25 +272,22 @@ def decode_segmap(image, nc=5):
if self.footprints[polyNo] is None:
continue

# Extract building footprint information and download the image viewing
# this footprint:
# Extract building footprint information :
footprint = np.fliplr(np.squeeze(np.array(self.footprints[polyNo])))

if self.streetScales[polyNo] is None:
continue


# Run image through the segmentation model
# Run building image through the segmentation model:
img = Image.open(self.street_images[polyNo])
imsize = img.size

trf = T.Compose([T.Resize(640),
trf = T.Compose([T.Resize(round(1000/max(imsize)*min(imsize))),
T.ToTensor(),
T.Normalize(mean = [0.485, 0.456, 0.406],
std = [0.229, 0.224, 0.225])])
std = [0.229, 0.224, 0.225])]) #

inp = trf(img).unsqueeze(0).to(dev)
scores = model.to(dev)(inp)['out']
pred = torch.argmax(scores.squeeze(), dim=0).detach().cpu().numpy()
predraw = torch.argmax(scores.squeeze(), dim=0).detach().cpu().numpy()
pred = np.array(Image.fromarray(np.uint8(predraw)).resize(imsize))

# Extract component masks
maskRoof = (pred==1).astype(np.uint8)
Expand All @@ -301,8 +301,7 @@ def decode_segmap(image, nc=5):
maskFacade = cv2.morphologyEx(openedFacadeMask, cv2.MORPH_CLOSE, kernel)
openedWinMask = cv2.morphologyEx(maskWin, cv2.MORPH_OPEN, kernel)
maskWin = cv2.morphologyEx(openedWinMask, cv2.MORPH_CLOSE, kernel)




# Find roof contours
contours, _ = cv2.findContours(maskRoof,cv2.RETR_EXTERNAL,
Expand Down Expand Up @@ -348,23 +347,72 @@ def decode_segmap(image, nc=5):
R0PixHeight = max(y)-min(y)
R1PixHeight = R0PixHeight + roofPixHeight

# Get the raw depthmap for the building and crop it so that it covers
# just the segmentation mask of the bbox for the building facade:
depthmap = self.depthmaps[polyNo]
depthmapbbox = depthmap.crop((min(x),min(y),max(x),max(y)))

# Calculate heigths of interest
ind = storymodel.system_dict['infer']['images'].index(self.street_images[polyNo])
normalizer = storymodel.system_dict['infer']['predictions'][ind]*14*(1.3-random()*0.4)
R0 = R0PixHeight*self.streetScales[polyNo]
if R0>normalizer:
normfact = normalizer/R0
R0 = normalizer
elif R0<0.8*normalizer:
normfact = normalizer/R0
R0 = normalizer
else:
normfact = 1
R1 = R1PixHeight*self.streetScales[polyNo]*normfact
if (R1-R0)>1.5*normalizer:
R1 = R0 + (1.2-random()*0.4)*normalizer
# Convert the depthmap to a Numpy array for further processing and
# sample the raw depthmap at its vertical centerline:
depthmapbbox_arr = np.asarray(depthmapbbox)
depthmap_cl = depthmapbbox_arr[:,round(depthmapbbox.size[0]/2)]

# Calculate the vertical camera angles corresponding to the bottom
# and top of the facade bounding box:
imHeight = img.size[1]
angleTop = ((imHeight/2 - min(y))/(imHeight/2))*math.pi/2
angleBottom = ((imHeight/2 - max(y))/(imHeight/2))*math.pi/2

# Take the first derivative of the depthmap with respect to vertical
# pixel location and identify the depthmap discontinuity (break)
# locations:
break_pts = [0]
boolval_prev = True
depthmap_cl_dx = np.append(abs(np.diff(depthmap_cl))<0.1,True)
for (counter, boolval_curr) in enumerate(depthmap_cl_dx):
if (boolval_prev==True and boolval_curr==False) or (boolval_prev==False and boolval_curr==True):
break_pts.append(counter)
boolval_prev = boolval_curr
break_pts.append(counter)

# Identify the depthmap segments to keep for extrapolation, i.e.,
# segments that are not discontinuities in the depthmap:
segments_keep = []
for i in range(len(break_pts)-1):
if all(depthmap_cl_dx[break_pts[i]:break_pts[i+1]]) and all(depthmap_cl[break_pts[i]:break_pts[i+1]]!=255):
segments_keep.append((break_pts[i],break_pts[i+1]))

# Fit line models to individual (kept) segments of the depthmap and
# determine the model that results in the smallest residual for
# all kept depthmap points:
lm = LinearRegression(fit_intercept = True)
x = np.arange(depthmapbbox.size[1])
xKeep = np.hstack([x[segment[0]:segment[1]] for segment in segments_keep])
yKeep = np.hstack([depthmap_cl[segment[0]:segment[1]] for segment in segments_keep])
residualprev = 1e10
model_lm = deepcopy(lm)
for segment in segments_keep:
xvect = x[segment[0]:segment[1]]
yvect = depthmap_cl[segment[0]:segment[1]]

# Fit model:
lm.fit(xvect.reshape(-1, 1),yvect)
preds = lm.predict(xKeep.reshape(-1,1))
residual = np.sum(np.square(yKeep-preds))
if residual<residualprev:
model_lm = deepcopy(lm)
residualprev = residual

# Extrapolate depthmap using the best-fit model:
depthmap_cl_depths = model_lm.predict(x.reshape(-1,1))

# Calculate heigths of interest:
R0 = (depthmap_cl_depths[0]*math.sin(angleTop)
- depthmap_cl_depths[-1]*math.sin(angleBottom))*3.28084
scale = R0/R0PixHeight
R1 = R1PixHeight*scale

# Calculate roof pitch:
roof_run = compute_roofrun(footprint)
roofPitch = (R1-R0)/roof_run
self.predictions.loc[polyNo] = [self.street_images[polyNo],
Expand Down
Loading

0 comments on commit b349f54

Please sign in to comment.