blender/intern/cycles/util/util_path.cpp

/*
 * Copyright 2011-2013 Blender Foundation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "util/util_debug.h"
#include "util/util_md5.h"
#include "util/util_path.h"
#include "util/util_string.h"

#include <OpenImageIO/filesystem.h>
#include <OpenImageIO/strutil.h>
#include <OpenImageIO/sysutil.h>

OIIO_NAMESPACE_USING

#include <stdio.h>

#include <sys/stat.h>

#if defined(_WIN32)
#  define DIR_SEP '\\'
#  define DIR_SEP_ALT '/'
#  include <direct.h>
#else
#  define DIR_SEP '/'
#  include <dirent.h>
#  include <pwd.h>
#  include <unistd.h>
#  include <sys/types.h>
#endif

#ifdef HAVE_SHLWAPI_H
#  include <shlwapi.h>
#endif

#include "util/util_map.h"
#include "util/util_windows.h"

CCL_NAMESPACE_BEGIN

#ifdef _WIN32
#  if defined(_MSC_VER) || defined(__MINGW64__)
typedef struct _stat64 path_stat_t;
#  elif defined(__MINGW32__)
typedef struct _stati64 path_stat_t;
#  else
typedef struct _stat path_stat_t;
#  endif
#  ifndef S_ISDIR
#    define S_ISDIR(x) (((x) & _S_IFDIR) == _S_IFDIR)
#  endif
#else
typedef struct stat path_stat_t;
#endif

static string cached_path = "";
static string cached_user_path = "";
static string cached_xdg_cache_path = "";

namespace {

#ifdef _WIN32
class directory_iterator {
public:
	class path_info {
	public:
		path_info(const string& path,
		          const WIN32_FIND_DATAW& find_data)
		: path_(path),
		  find_data_(find_data)
		{
		}

		string path() {
			return path_join(path_, string_from_wstring(find_data_.cFileName));
		}
	protected:
		const string& path_;
		const WIN32_FIND_DATAW& find_data_;
	};

	directory_iterator()
	: path_info_("", find_data_),
	  h_find_(INVALID_HANDLE_VALUE)
	{
	}

	explicit directory_iterator(const string& path)
	: path_(path),
	  path_info_(path, find_data_)
	{
		string wildcard = path;
		if(wildcard[wildcard.size() - 1] != DIR_SEP) {
			wildcard += DIR_SEP;
		}
		wildcard += "*";
		h_find_ = FindFirstFileW(string_to_wstring(wildcard).c_str(),
		                         &find_data_);
		if(h_find_ != INVALID_HANDLE_VALUE) {
			skip_dots();
		}
	}

	~directory_iterator()
	{
		if(h_find_ != INVALID_HANDLE_VALUE) {
			FindClose(h_find_);
		}
	}

	directory_iterator& operator++()
	{
		step();
		return *this;
	}

	path_info* operator-> ()
	{
		return &path_info_;
	}

	bool operator!=(const directory_iterator& other)
	{
		return h_find_ != other.h_find_;
	}

protected:
	bool step()
	{
		if(do_step()) {
			return skip_dots();
		}
		return false;
	}

	bool do_step()
	{
		if(h_find_ != INVALID_HANDLE_VALUE) {
			bool result = FindNextFileW(h_find_, &find_data_) == TRUE;
			if(!result) {
				FindClose(h_find_);
				h_find_ = INVALID_HANDLE_VALUE;
			}
			return result;
		}
		return false;
	}

	bool skip_dots()
	{
		while(wcscmp(find_data_.cFileName, L".") == 0 ||
		      wcscmp(find_data_.cFileName, L"..") == 0)
		{
			if(!do_step()) {
				return false;
			}
		}
		return true;
	}

	string path_;
	path_info path_info_;
	WIN32_FIND_DATAW find_data_;
	HANDLE h_find_;
};
#else  /* _WIN32 */

class directory_iterator {
public:
	class path_info {
	public:
		explicit path_info(const string& path)
		: path_(path),
		  entry_(NULL)
		{
		}

		string path() {
			return path_join(path_, entry_->d_name);
		}

		void current_entry_set(const struct dirent *entry)
		{
			entry_ = entry;
		}
	protected:
		const string& path_;
		const struct dirent *entry_;
	};

	directory_iterator()
	: path_info_(""),
	  name_list_(NULL),
	  num_entries_(-1),
	  cur_entry_(-1)
	{
	}

	explicit directory_iterator(const string& path)
	: path_(path),
	  path_info_(path_),
	  cur_entry_(0)
	{
		num_entries_ = scandir(path.c_str(),
		                       &name_list_,
		                       NULL,
		                       alphasort);
		if(num_entries_ < 0) {
			perror("scandir");
		}
		else {
			skip_dots();
		}
	}

	~directory_iterator()
	{
		destroy_name_list();
	}

	directory_iterator& operator++()
	{
		step();
		return *this;
	}

	path_info* operator-> ()
	{
		path_info_.current_entry_set(name_list_[cur_entry_]);
		return &path_info_;
	}

	bool operator!=(const directory_iterator& other)
	{
		return name_list_ != other.name_list_;
	}

protected:
	bool step()
	{
		if(do_step()) {
			return skip_dots();
		}
		return false;
	}

	bool do_step()
	{
		++cur_entry_;
		if(cur_entry_ >= num_entries_) {
			destroy_name_list();
			return false;
		}
		return true;
	}

	/* Skip . and .. folders. */
	bool skip_dots()
	{
		while(strcmp(name_list_[cur_entry_]->d_name, ".") == 0 ||
		      strcmp(name_list_[cur_entry_]->d_name, "..") == 0)
		{
			if(!step()) {
				return false;
			}
		}
		return true;
	}

	void destroy_name_list()
	{
		if(name_list_ == NULL) {
			return;
		}
		for(int i = 0; i < num_entries_; ++i) {
			free(name_list_[i]);
		}
		free(name_list_);
		name_list_ = NULL;
	}

	string path_;
	path_info path_info_;
	struct dirent **name_list_;
	int num_entries_, cur_entry_;
};

#endif  /* _WIN32 */

size_t find_last_slash(const string& path)
{
	for(size_t i = 0; i < path.size(); ++i) {
		size_t index = path.size() - 1 - i;
#ifdef _WIN32
		if(path[index] == DIR_SEP || path[index] == DIR_SEP_ALT)
#else
		if(path[index] == DIR_SEP)
#endif
		{
			return index;
		}
	}
	return string::npos;
}

}  /* namespace */

static char *path_specials(const string& sub)
{
	static bool env_init = false;
	static char *env_shader_path;
	static char *env_source_path;
	if(!env_init) {
		env_shader_path = getenv("CYCLES_SHADER_PATH");
		/* NOTE: It is KERNEL in env variable for compatibility reasons. */
		env_source_path = getenv("CYCLES_KERNEL_PATH");
		env_init = true;
	}
	if(env_shader_path != NULL && sub == "shader") {
		return env_shader_path;
	}
	else if(env_shader_path != NULL && sub == "source") {
		return env_source_path;
	}
	return NULL;
}

#if defined(__linux__) || defined(__APPLE__)
static string path_xdg_cache_get()
{
	const char *home = getenv("XDG_CACHE_HOME");
	if(home) {
		return string(home);
	}
	else {
		home = getenv("HOME");
		if(home == NULL) {
			home = getpwuid(getuid())->pw_dir;
		}
		return path_join(string(home), ".cache");
	}
}
#endif

void path_init(const string& path, const string& user_path)
{
	cached_path = path;
	cached_user_path = user_path;

#ifdef _MSC_VER
	// workaround for https://svn.boost.org/trac/boost/ticket/6320
	// indirectly init boost codec here since it's not thread safe, and can
	// cause crashes when it happens in multithreaded image load
	OIIO::Filesystem::exists(path);
#endif
}

string path_get(const string& sub)
{
	char *special = path_specials(sub);
	if(special != NULL)
		return special;

	if(cached_path == "")
		cached_path = path_dirname(Sysutil::this_program_path());

	return path_join(cached_path, sub);
}

string path_user_get(const string& sub)
{
	if(cached_user_path == "")
		cached_user_path = path_dirname(Sysutil::this_program_path());

	return path_join(cached_user_path, sub);
}

string path_cache_get(const string& sub)
{
#if defined(__linux__) || defined(__APPLE__)
	if(cached_xdg_cache_path == "") {
		cached_xdg_cache_path = path_xdg_cache_get();
	}
	string result = path_join(cached_xdg_cache_path, "cycles");
	return path_join(result, sub);
#else
	/* TODO(sergey): What that should be on Windows? */
	return path_user_get(path_join("cache", sub));
#endif
}

#if defined(__linux__) || defined(__APPLE__)
string path_xdg_home_get(const string& sub = "");
#endif

string path_filename(const string& path)
{
	size_t index = find_last_slash(path);
	if(index != string::npos) {
		/* Corner cases to match boost behavior. */
#ifndef _WIN32
		if(index == 0 && path.size() == 1) {
			return path;
		}
#endif
		if(index == path.size() - 1) {
#ifdef _WIN32
			if(index == 2) {
				return string(1, DIR_SEP);
			}
#endif
			return ".";
		}
		return path.substr(index + 1, path.size() - index - 1);
	}
	return path;
}

string path_dirname(const string& path)
{
	size_t index = find_last_slash(path);
	if(index != string::npos) {
#ifndef _WIN32
		if(index == 0 && path.size() > 1) {
			return string(1, DIR_SEP);
		}
#endif
		return path.substr(0, index);
	}
	return "";
}

string path_join(const string& dir, const string& file)
{
	if(dir.size() == 0) {
		return file;
	}
	if(file.size() == 0) {
		return dir;
	}
	string result = dir;
#ifndef _WIN32
	if(result[result.size() - 1] != DIR_SEP &&
	   file[0] != DIR_SEP)
#else
	if(result[result.size() - 1] != DIR_SEP &&
	   result[result.size() - 1] != DIR_SEP_ALT &&
	   file[0] != DIR_SEP &&
	   file[0] != DIR_SEP_ALT)
#endif
	{
		result += DIR_SEP;
	}
	result += file;
	return result;
}

string path_escape(const string& path)
{
	string result = path;
	string_replace(result, " ", "\\ ");
	return result;
}

bool path_is_relative(const string& path)
{
#ifdef _WIN32
#  ifdef HAVE_SHLWAPI_H
	return PathIsRelative(path.c_str());
#  else  /* HAVE_SHLWAPI_H */
	if(path.size() >= 3) {
		return !(((path[0] >= 'a' && path[0] <= 'z') ||
		         (path[0] >= 'A' && path[0] <= 'Z')) &&
		         path[1] == ':' && path[2] == DIR_SEP);
	}
	return true;
#  endif  /* HAVE_SHLWAPI_H */
#else  /* _WIN32 */
	if(path.size() == 0) {
		return 1;
	}
	return path[0] != DIR_SEP;
#endif  /* _WIN32 */
}

#ifdef _WIN32
/* Add a slash if the UNC path points to a share. */
static string path_unc_add_slash_to_share(const string& path)
{
	size_t slash_after_server = path.find(DIR_SEP, 2);
	if(slash_after_server != string::npos) {
		size_t slash_after_share = path.find(DIR_SEP,
		                                     slash_after_server + 1);
		if(slash_after_share == string::npos) {
			return path + DIR_SEP;
		}
	}
	return path;
}

/* Convert:
 *    \\?\UNC\server\share\folder\... to \\server\share\folder\...
 *    \\?\C:\ to C:\ and \\?\C:\folder\... to C:\folder\...
 */
static string path_unc_to_short(const string& path)
{
	size_t len = path.size();
	if((len > 3) &&
	   (path[0] ==  DIR_SEP) &&
	   (path[1] ==  DIR_SEP) &&
	   (path[2] ==  '?') &&
	   ((path[3] ==  DIR_SEP) || (path[3] ==  DIR_SEP_ALT)))
	{
		if((len > 5) && (path[5] ==  ':')) {
			return path.substr(4, len - 4);
		}
		else if((len > 7) &&
		        (path.substr(4, 3) == "UNC") &&
		        ((path[7] ==  DIR_SEP) || (path[7] ==  DIR_SEP_ALT)))
		{
			return "\\\\" + path.substr(8, len - 8);
		}
	}
	return path;
}

static string path_cleanup_unc(const string& path)
{
	string result = path_unc_to_short(path);
	if(path.size() > 2) {
		/* It's possible path is now a non-UNC. */
		if(result[0] == DIR_SEP && result[1] == DIR_SEP) {
			return path_unc_add_slash_to_share(result);
		}
	}
	return result;
}

/* Make path compatible for stat() functions. */
static string path_make_compatible(const string& path)
{
	string result = path;
	/* In Windows stat() doesn't recognize dir ending on a slash. */
	if(result.size() > 3 && result[result.size() - 1] == DIR_SEP) {
		result.resize(result.size() - 1);
	}
	/* Clean up UNC path. */
	if((path.size() >= 3) && (path[0] == DIR_SEP) && (path[1] == DIR_SEP)) {
		result = path_cleanup_unc(result);
	}
	/* Make sure volume-only path ends up wit ha directory separator. */
	if(result.size() == 2 && result[1] == ':') {
		result += DIR_SEP;
	}
	return result;
}

static int path_wstat(const wstring& path_wc, path_stat_t *st)
{
#if defined(_MSC_VER) || defined(__MINGW64__)
	return _wstat64(path_wc.c_str(), st);
#elif defined(__MINGW32__)
	return _wstati64(path_wc.c_str(), st);
#else
	return _wstat(path_wc.c_str(), st);
#endif
}

static int path_stat(const string& path, path_stat_t *st)
{
	wstring path_wc = string_to_wstring(path);
	return path_wstat(path_wc, st);
}
#else  /* _WIN32 */
static int path_stat(const string& path, path_stat_t *st)
{
	return stat(path.c_str(), st);
}
#endif  /* _WIN32 */

size_t path_file_size(const string& path)
{
	path_stat_t st;
	if(path_stat(path, &st) != 0) {
		return -1;
	}
	return st.st_size;
}

bool path_exists(const string& path)
{
#ifdef _WIN32
	string fixed_path = path_make_compatible(path);
	wstring path_wc = string_to_wstring(fixed_path);
	path_stat_t st;
	if(path_wstat(path_wc, &st) != 0) {
		return false;
	}
	return st.st_mode != 0;
#else  /* _WIN32 */
	struct stat st;
	if(stat(path.c_str(), &st) != 0) {
		return 0;
	}
	return st.st_mode != 0;
#endif /* _WIN32 */
}

bool path_is_directory(const string& path)
{
	path_stat_t st;
	if(path_stat(path, &st) != 0) {
		return false;
	}
	return S_ISDIR(st.st_mode);
}

static void path_files_md5_hash_recursive(MD5Hash& hash, const string& dir)
{
	if(path_exists(dir)) {
		directory_iterator it(dir), it_end;

		for(; it != it_end; ++it) {
			if(path_is_directory(it->path())) {
				path_files_md5_hash_recursive(hash, it->path());
			}
			else {
				string filepath = it->path();

				hash.append((const uint8_t*)filepath.c_str(), filepath.size());
				hash.append_file(filepath);
			}
		}
	}
}

string path_files_md5_hash(const string& dir)
{
	/* computes md5 hash of all files in the directory */
	MD5Hash hash;

	path_files_md5_hash_recursive(hash, dir);

	return hash.get_hex();
}

static bool create_directories_recursivey(const string& path)
{
	if(path_is_directory(path)) {
		/* Directory already exists, nothing to do. */
		return true;
	}
	if(path_exists(path)) {
		/* File exists and it's not a directory. */
		return false;
	}

	string parent = path_dirname(path);
	if(parent.size() > 0 && parent != path) {
		if(!create_directories_recursivey(parent)) {
			return false;
		}
	}

#ifdef _WIN32
	wstring path_wc = string_to_wstring(path);
	return _wmkdir(path_wc.c_str()) == 0;
#else
	return mkdir(path.c_str(), 0777) == 0;
#endif
}

void path_create_directories(const string& filepath)
{
	string path = path_dirname(filepath);
	create_directories_recursivey(path);
}

bool path_write_binary(const string& path, const vector<uint8_t>& binary)
{
	path_create_directories(path);

	/* write binary file from memory */
	FILE *f = path_fopen(path, "wb");

	if(!f)
		return false;

	if(binary.size() > 0)
		fwrite(&binary[0], sizeof(uint8_t), binary.size(), f);

	fclose(f);

	return true;
}

bool path_write_text(const string& path, string& text)
{
	vector<uint8_t> binary(text.length(), 0);
	std::copy(text.begin(), text.end(), binary.begin());

	return path_write_binary(path, binary);
}

bool path_read_binary(const string& path, vector<uint8_t>& binary)
{
	/* read binary file into memory */
	FILE *f = path_fopen(path, "rb");

	if(!f) {
		binary.resize(0);
		return false;
	}

	binary.resize(path_file_size(path));

	if(binary.size() == 0) {
		fclose(f);
		return false;
	}

	if(fread(&binary[0], sizeof(uint8_t), binary.size(), f) != binary.size()) {
		fclose(f);
		return false;
	}

	fclose(f);

	return true;
}

bool path_read_text(const string& path, string& text)
{
	vector<uint8_t> binary;

	if(!path_exists(path) || !path_read_binary(path, binary))
		return false;

	const char *str = (const char*)&binary[0];
	size_t size = binary.size();
	text = string(str, size);

	return true;
}

uint64_t path_modified_time(const string& path)
{
	path_stat_t st;
	if(path_stat(path, &st) != 0) {
		return 0;
	}
	return st.st_mtime;
}

bool path_remove(const string& path)
{
	return remove(path.c_str()) == 0;
}

struct SourceReplaceState {
	typedef map<string, string> ProcessedMapping;
	/* Base director for all relative include headers. */
	string base;
	/* Result of processed files. */
	ProcessedMapping processed_files;
	/* Set of files which are considered "precompiled" and which are replaced
	 * with and empty string on a subsequent occurrence in include statement.
	 */
	set<string> precompiled_headers;
};

static string path_source_replace_includes_recursive(
        const string& source,
        const string& source_filepath,
        SourceReplaceState *state);

static string line_directive(const SourceReplaceState& state,
                             const string& path,
                             const int line)
{
	string unescaped_path = path;
	/* First we make path relative. */
	if(string_startswith(unescaped_path, state.base.c_str())) {
		const string base_file = path_filename(state.base);
		const size_t base_len = state.base.length();
		unescaped_path = base_file +
		        unescaped_path.substr(base_len,
		                            unescaped_path.length() - base_len);
	}
	/* Second, we replace all unsafe characters. */
	const size_t length = unescaped_path.length();
	string escaped_path = "";
	for(size_t i = 0; i < length; ++i) {
		const char ch = unescaped_path[i];
		if(strchr("\"\'\?\\", ch) != NULL) {
			escaped_path += "\\";
		}
		escaped_path += ch;
	}
	/* TODO(sergey): Check whether using std::to_string combined with several
	 * concatenation operations is any faster.
	 */
	return string_printf("#line %d \"%s\"", line, escaped_path.c_str());
}

static string path_source_handle_preprocessor(
        const string& preprocessor_line,
        const string& source_filepath,
        const size_t line_number,
        SourceReplaceState *state)
{
	string result = preprocessor_line;
	string token = string_strip(
	        preprocessor_line.substr(1, preprocessor_line.size() - 1));
	if(string_startswith(token, "include")) {
		token = string_strip(token.substr(7, token.size() - 7));
		if(token[0] == '"') {
			const size_t n_start = 1;
			const size_t n_end = token.find("\"", n_start);
			const string filename = token.substr(n_start, n_end - n_start);
			const bool is_precompiled = string_endswith(token, "// PRECOMPILED");
			string filepath = path_join(state->base, filename);
			if(!path_exists(filepath)) {
				filepath = path_join(path_dirname(source_filepath),
				                     filename);
			}
			if(is_precompiled) {
				state->precompiled_headers.insert(filepath);
			}
			string text;
			if(path_read_text(filepath, text)) {
				text = path_source_replace_includes_recursive(
				        text, filepath, state);
				/* Use line directives for better error messages. */
				result = line_directive(*state, filepath, 1) + "\n"
				     + text + "\n"
				     + line_directive(*state, source_filepath, line_number + 1);
			}
		}
	}
	return result;
}

/* Our own little c preprocessor that replaces #includes with the file
 * contents, to work around issue of OpenCL drivers not supporting
 * include paths with spaces in them.
 */
static string path_source_replace_includes_recursive(
        const string& source,
        const string& source_filepath,
        SourceReplaceState *state)
{
	/* Try to re-use processed file without spending time on replacing all
	 * include directives again.
	 */
	SourceReplaceState::ProcessedMapping::iterator replaced_file =
	        state->processed_files.find(source_filepath);
	if(replaced_file != state->processed_files.end()) {
		if(state->precompiled_headers.find(source_filepath) !=
		        state->precompiled_headers.end()) {
			return "";
		}
		return replaced_file->second;
	}
	/* Perform full file processing. */
	string result = "";
	const size_t source_length = source.length();
	size_t index = 0;
	/* Information about where we are in the source. */
	size_t line_number = 0, column_number = 1;
	/* Currently gathered non-preprocessor token.
	 * Store as start/length rather than token itself to avoid overhead of
	 * memory re-allocations on each character concatenation.
	 */
	size_t token_start = 0, token_length = 0;
	/* Denotes whether we're inside of preprocessor line, together with
	 * preprocessor line itself.
	 *
	 * TODO(sergey): Investigate whether using token start/end position
	 * gives measurable speedup.
	 */
	bool inside_preprocessor = false;
	string preprocessor_line = "";
	/* Actual loop over the whole source. */
	while(index < source_length) {
		const char ch = source[index];
		if(ch == '\n') {
			if(inside_preprocessor) {
				result += path_source_handle_preprocessor(preprocessor_line,
				                                          source_filepath,
				                                          line_number,
				                                          state);
				/* Start gathering net part of the token. */
				token_start = index;
				token_length = 0;
			}
			inside_preprocessor = false;
			preprocessor_line = "";
			column_number = 0;
			++line_number;
		}
		else if(ch == '#' && column_number == 1 && !inside_preprocessor) {
			/* Append all possible non-preprocessor token to the result. */
			if(token_length != 0) {
				result.append(source, token_start, token_length);
				token_start = index;
				token_length = 0;
			}
			inside_preprocessor = true;
		}
		if(inside_preprocessor) {
			preprocessor_line += ch;
		}
		else {
			++token_length;
		}
		++index;
		++column_number;
	}
	/* Append possible tokens which happened before special events handled
	 * above.
	 */
	if(token_length != 0) {
		result.append(source, token_start, token_length);
	}
	if(inside_preprocessor) {
		result += path_source_handle_preprocessor(preprocessor_line,
		                                          source_filepath,
		                                          line_number,
		                                          state);
	}
	/* Store result for further reuse. */
	state->processed_files[source_filepath] = result;
	return result;
}

string path_source_replace_includes(const string& source,
                                    const string& path,
                                    const string& source_filename)
{
	SourceReplaceState state;
	state.base = path;
	return path_source_replace_includes_recursive(
	        source,
	        path_join(path, source_filename),
	        &state);
}

FILE *path_fopen(const string& path, const string& mode)
{
#ifdef _WIN32
	wstring path_wc = string_to_wstring(path);
	wstring mode_wc = string_to_wstring(mode);
	return _wfopen(path_wc.c_str(), mode_wc.c_str());
#else
	return fopen(path.c_str(), mode.c_str());
#endif
}

void path_cache_clear_except(const string& name, const set<string>& except)
{
	string dir = path_user_get("cache");

	if(path_exists(dir)) {
		directory_iterator it(dir), it_end;

		for(; it != it_end; ++it) {
			string filename = path_filename(it->path());

			if(string_startswith(filename, name.c_str()))
				if(except.find(filename) == except.end())
					path_remove(it->path());
		}
	}

}

CCL_NAMESPACE_END