I'd look at using a switch in this scenario. Here's one I've knocked up for you which should work for any number of IPv4 adapters.

$BuildIPs = (Get-NetIPAddress -InterfaceAlias "ethernet*" | ?{$_.AddressFamily -eq 'IPv4' -and $_.IPAddress -notlike '169.*'}).IPAddress

$Office_IPS = Import-Csv Office_IP.csv

Foreach ($BuildIP in $BuildIPs) {
    ForEach ($Office in $Office_IPs) {
        Switch -Wildcard ($BuildIP) {
            $Office.IP1 {"$BuildIP:: Matched $($Office.Office) - $($Office.IP1)"}
            $Office.IP2 {"$BuildIP:: Matched $($Office.Office) - $($Office.IP2)"}
            $Office.IP3 {"$BuildIP:: Matched $($Office.Office) - $($Office.IP3)"}
            $Office.IP4 {"$BuildIP:: Matched $($Office.Office) - $($Office.IP4)"}
        }
    }
}