Introduction
git clone git@github.com:Mathpix/api-examples.git
cd api-examples/images
Welcome to the Mathpix API! You send us an image, we tell you about the math that’s in it. It’s that simple. We return latex as well as image metadata that you can use in your application. To see examples of images our engine can recognize, see the following link:
We have examples of how to call our API for Shell, Python, Android, and iOS. You can view code examples in the dark area to the right, and you can switch the programming language of the examples with the tabs in the top right.
Note for iOS developers: if you want to integrate Mathpix into your iOS app, there is no easier way than using the Mathpix SDK for iOS: https://github.com/Mathpix/ios-client. With this SDK you can integrate Mathpix into your app with one single line of code without having to implement camera functionality. Also, the UI is fully customizable.
If you have any problems, please send us an email at support@mathpix.com.
Translations
Processing a single image
Call the API as follows:
#!/usr/bin/env python
import sys
import base64
import requests
import json
# put desired file path here
file_path = 'limit.jpg'
image_uri = "data:image/jpg;base64," + base64.b64encode(open(file_path, "rb").read())
# (python3) image_uri = "data:image/jpg;base64," + base64.b64encode(open(file_path, "rb").read()).decode()
r = requests.post("https://api.mathpix.com/v3/latex",
data=json.dumps({'src': image_uri}),
headers={"app_id": "trial", "app_key": "34f1a4cea0eaca8540c95908b4dc84ab",
"Content-type": "application/json"})
print json.dumps(json.loads(r.text), indent=4, sort_keys=True)
curl -X POST https://api.mathpix.com/v3/latex \
-H 'app_id: trial' \
-H 'app_key: 34f1a4cea0eaca8540c95908b4dc84ab' \
-H 'Content-Type: application/json' \
--data '{ "src": "data:image/jpeg;base64,'$(base64 -i limit.jpg)'" }'
// Using OKHttp
// git url : https://github.com/Mathpix/api-examples/tree/extra/android
OkHttpClient client = new OkHttpClient();
MediaType mediaType = MediaType.parse("application/json");
RequestBody body = RequestBody.create(mediaType, "{ \"src\" : \"data:image/jpeg;base64,{BASE64-STRING}\" }");
Request request = new Request.Builder()
.url("https://api.mathpix.com/v3/latex")
.addHeader("content-type", "application/json")
.addHeader("app_id", "mathpix")
.addHeader("app_key", "34f1a4cea0eaca8540c95908b4dc84ab")
.post(body)
.build();
Response response = client.newCall(request).execute();
Using NSUrl
import Foundation
let headers = [
"content-type": "application/json",
"app_id": "mathpix",
"app_key": "34f1a4cea0eaca8540c95908b4dc84ab"
]
let parameters = ["src": "data:image/jpeg;base64,{BASE64-STRING}"] as [String : Any]
let postData = JSONSerialization.data(withJSONObject: parameters, options: [])
let request = NSMutableURLRequest(url: NSURL(string: "https://api.mathpix.com/v3/latex")! as URL,
cachePolicy: .useProtocolCachePolicy,
timeoutInterval: 60.0)
request.httpMethod = "POST"
request.allHTTPHeaderFields = headers
request.httpBody = postData as Data
let session = URLSession.shared
let dataTask = session.dataTask(with: request as URLRequest, completionHandler: { (data, response, error) -> Void in
if (error != nil) {
print(error)
} else {
let httpResponse = response as? HTTPURLResponse
print(httpResponse)
}
})
dataTask.resume()
// Using Alamofire
// git url : https://github.com/Mathpix/api-examples/tree/extra/swift
func processSingleImage(imageName : String) {
if let data = NSData(contentsOfFile: imageName) {
let base64String = data.base64EncodedString(options: .init(rawValue: 0))
let parameters : Parameters = [
"src" : "data:image/jpeg;base64," + base64String
]
Alamofire.request("https://api.mathpix.com/v3/latex",
method: .post,
parameters : parameters,
encoding: JSONEncoding.default,
headers: [
"app_id" : "mathpix",
"app_key" : "139ee4b61be2e4abcfb1238d9eb99902"
])
.responseJSON{ response in
if let JSON = response.result.value {
print("\(JSON)")
}
}
}
}
Using NSUrl
#import <Foundation/Foundation.h>
NSDictionary *headers = @{ @"content-type": @"application/json",
@"app_id": @"mathpix",
@"app_key": @"34f1a4cea0eaca8540c95908b4dc84ab"};
NSDictionary *parameters = @{ @"src": @"data:image/jpeg;base64,{BASE64-STRING}" };
NSData *postData = [NSJSONSerialization dataWithJSONObject:parameters options:0 error:nil];
NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:[NSURL URLWithString:@"https://api.mathpix.com/v3/latex"]
cachePolicy:NSURLRequestUseProtocolCachePolicy
timeoutInterval:60.0];
[request setHTTPMethod:@"POST"];
[request setAllHTTPHeaderFields:headers];
[request setHTTPBody:postData];
NSURLSession *session = [NSURLSession sharedSession];
NSURLSessionDataTask *dataTask = [session dataTaskWithRequest:request
completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) {
if (error) {
NSLog(@"%@", error);
} else {
NSHTTPURLResponse *httpResponse = (NSHTTPURLResponse *) response;
NSLog(@"%@", httpResponse);
}
}];
[dataTask resume];
// Using AFNetworking
// git url : https://github.com/Mathpix/api-examples/tree/extra/objc
- (void)processSingleImage: (NSString *)imageName {
NSData *data = [NSData dataWithContentsOfFile:imageName options:NSDataReadingMapped error:nil];
NSString *base64String = [data base64EncodedStringWithOptions:0];
NSDictionary *param = @{@"src" : [NSString stringWithFormat:@"data:image/jpeg;base64,%@", base64String]};
NSMutableURLRequest *request = [[AFJSONRequestSerializer serializer] requestWithMethod:@"POST" URLString:@"https://api.mathpix.com/v3/latex" parameters:param error:nil];
[request setValue:@"mathpix" forHTTPHeaderField:@"app_id"];
[request setValue:@"139ee4b61be2e4abcfb1238d9eb99902" forHTTPHeaderField:@"app_key"];
AFURLSessionManager *manager = [[AFURLSessionManager alloc] initWithSessionConfiguration:[NSURLSessionConfiguration defaultSessionConfiguration]];
NSURLSessionUploadTask *uploadTask;
uploadTask = [manager
uploadTaskWithStreamedRequest:request
progress:^(NSProgress * _Nonnull uploadProgress) {
dispatch_async(dispatch_get_main_queue(), ^{
});
}
completionHandler:^(NSURLResponse * _Nonnull response, id _Nullable responseObject, NSError * _Nullable error) {
if (error) {
NSLog(@"Error: %@", error);
} else { //SUCCESS
NSLog(@"Response: %@", responseObject);
}
}];
[uploadTask resume];
}
The API returns JSON structured as show here:
{
"detection_list": [],
"detection_map": {
"contains_chart": 0,
"contains_diagram": 0,
"contains_geometry": 0,
"contains_graph": 0,
"contains_table": 0,
"is_inverted": 0,
"is_not_math": 0,
"is_printed": 0
},
"error": "",
"latex": "\\lim _ { x \\rightarrow 3 } ( \\frac { x ^ { 2 } + 9 } { x - 3 } )",
"latex_confidence": 0.86757309488734,
"position": {
"height": 273,
"top_left_x": 57,
"top_left_y": 14,
"width": 605
}
}
Request parameters
An API request must contain two headers (app_id and app_key) and a JSON body with a src field that specifies an image as a string either containing a public resource reference to the image content or a data uri containing a base64 encoding of the image. The request body may also contain additional optional fields.
The region field, if present, specifies the image area to process using the properties top_left_x, top_left_y, width, and height, which are all in pixel coordinates (integer valued). For example,
“region”: {
“top_left_x”: 0, “top_left_y”: 0, “width”: 10, “height”: 10
}
specifies a 10 x 10 region at the top left of the image.
The optional ocr field indicates what kinds of content to recognize.
The default value is ["math"]. An app that wants to process text
should pass ["math", "text"]. Text is returned inside a \text{} environment
so that the result can be display using a LaTeX renderer
such as MathJax or KaTeX.
The confidence_threshold field, if present, sets the minimum acceptable confidence that the expression is 100% correct. If the expression receives a confidence value lower than this number, the result contains an error field and no latex output.
The confidence_rate_threshold field, if present, sets the minimum acceptable acceptable average per character confidence. This field should be used instead of confidence_threshold in applications where the overall accuracy of the system is more important than the accuracy in recognizing the images in question without a single defect. For example, an auto Latex system is useful even if there are a few characters which are not in it’s vocabulary. As long as the algorithm is generally confident, a few missed characters are acceptable as the user can correct these rapidly.
The formats field, if present, contains an array of one or more strings
specifying the desired representations for the results.
Accepted representations are
latex_normal for the most direct representation of the input,
latex_simplified for modifications that assist symbolic processing
such as shortening operator names, replacing long division with a fraction, and
converting a column of operands into a single formula;
latex_styled for modifications to improve the visual appearance such as
adding ’\left’ and ’\right’ around parenthesized expressions
that contain tall expressions like subscript or superscript;
latex_list to split output into a list of simplified strings
that assist in processing multiple equations,
text to return LaTeX using text mode at the top level instead of math mode,
mathml to return a string containing the MathML for the recognized math; and
wolfram to return a string that is compatible with the Wolfram Alpha engine.
For the mathml and wolfram formats the server will add
a mathml_error or wolfram_error field if unable to produce
MathML or compatible Wolfram Alpha output from the latex.
If formats is not present then the result will contain a latex field
with either the latex_normal or latex_simplified representation,
depending on the app_key.
Note that an app using the text format must also specify
the ocr option as [‘math’,'text’]. The returned string has \text{}
removed and \( and \) added around math. For example, if the latex_normal
string is \text { The answer is } f ( x ) = 3 x
then the result form the text format will be
The answer is \( f ( x ) = 3 x \).
The text format also flattens simple arrays such as might result
from a text paragraph.
For backward compatibility the server also accepts url as an alternative to src and a formats field containing latex, latex_confidence_threshold, and latex_confidence_rate_threshold fields. The result for such requests contains a latex field with the specified latex format.
The skip_recrop field indicates whether Mathpix should skip
trying to crop out irrelevant parts of the image. By default,
it will not do so. This is to benefit apps for which input images
contain lots of noise that should be filtered. If images do not
contain noise, skip_recrop should be set to True.
| Parameter | Description |
|---|---|
| app_id | Application identifier (e.g. mymathsolver) |
| app_key | Application key |
| content_type | application/json |
| src | string |
| region? | {top_left_x, top_left_y, width, height} |
| ocr? | [“math”?, “text”?] |
| skip_recrop? | bool |
| confidence_threshold? | number in [0,1] |
| confidence_rate_threshold | number in [0,1] |
| formats? | string[] |
Receiving results
The server responds with the results for the image unless the request
contains a callback object, in which case the response only contains
a single session_id field and the server asynchronously sends the result
to the url specified by callback.post. The asynchronous post contains
the result object and a reply object containing the session_id.
An application may override the session_id response by specifying a value for
callback.reply in the request. The server will use this value
both in the response and the asynchronous post.
Applications that wish to provide additional data in the post may specify
a callback.body value. The server will pass this value to the post.
If callback.headers is provided then the server will send the contents
as headers in the post.
You can create a callback URL for testing simply by visiting https://RequestBin.com/. Sample code to test the async API option are on the right.
To test the async API, replace the callback URL below with your own by visiting https://RequestBin.com/.
#!/usr/bin/env python
import sys
import base64
import requests
import json
file_path = "integral.jpg"
image_uri = "data:image/jpeg;base64," + base64.b64encode(open(file_path, "rb").read())
# (python3) image_uri = "data:image/jpg;base64," + base64.b64encode(open(file_path, "rb").read()).decode()
r = requests.post("https://api.mathpix.com/v3/latex",
data=json.dumps({'url': image_uri, 'callback': {'post': 'http://RequestBin.com/10z3g561', 'reply': file_path}}),
headers={"app_id": "trial", "app_key": "34f1a4cea0eaca8540c95908b4dc84ab",
"Content-type": "application/json"})
print r.text
curl -X POST https://api.mathpix.com/v3/latex \
-H 'app_id: trial' \
-H 'app_key: 34f1a4cea0eaca8540c95908b4dc84ab' \
-H 'Content-Type: application/json' \
--data '{ "src": "data:image/jpeg;base64,'$(base64 -i integral.jpg)'","callback":{"post": "http://RequestBin.com/10z3g561", "reply": "integral.jpg"}}'
The following JSON is delivered to the callback URL via a POST request:
{
"reply": "integral.jpg",
"result": {
"detection_list": [
"is_printed"
],
"detection_map": {
"contains_chart": 0,
"contains_diagram": 0,
"contains_geometry": 0,
"contains_graph": 0,
"contains_table": 0,
"is_inverted": 0,
"is_not_math": 0,
"is_printed": 1
},
"error": "",
"latex": "\\int \\frac { 4 x } { \\sqrt { x ^ { 2 } + 1 } } d x",
"latex_confidence": 0.99817252453161,
"latex_list": [],
"position": {
"height": 215,
"top_left_x": 57,
"top_left_y": 0,
"width": 605
}
}
}
Results
Example output JSON for the following image is shown on the right.
Confidence values
The API returns confidence scores for the returned latex in the latex_confidence field. It is recommended that you discard results that are below a certain confidence value. You can tune this threshold value for your application. We use a 20% confidence cutoff for our iOS and Android demo apps.
Detection types
The API defines multiple detection types to the client inside the JSON field detection_map. The fields are described below:
| Detection | Definition |
|---|---|
| contains_chart | Contains a visual representation of a discrete dataset. |
| contains_table | Contains a tabular representation of a discrete dataset. |
| contains_diagram | Contains a diagram. |
| contains_graph | Contains a 1D, 2D, or 3D graph. |
| contains_geometry | Contains chart, table, diagram, or graph. |
| is_printed | The image is taken of printed math, not handwritten math. |
| is_not_math | No valid equation was detected. |
The API returns confidence values for each of these detection categories.
Detection coordinates
The position field contains the bounding box of the equation of interest, in pixel coordinates of the image sent. Note that we consider systems of algebraic equations and multiple line equations as single equations. If multiple equations are separated by a lot of text, we will return the first equation only.
Processing a batch of images
A batch request is made as follows:
curl -X POST https://api.mathpix.com/v3/batch \
-H "app_id: trial" \
-H "app_key: 34f1a4cea0eaca8540c95908b4dc84ab" \
-H "Content-Type: application/json" \
--data '{ "urls": {"inverted": "https://raw.githubusercontent.com/Mathpix/api-examples/master/images/inverted.jpg", "algebra": "https://raw.githubusercontent.com/Mathpix/api-examples/master/images/algebra.jpg"},"callback":{"post": "http://RequestBin.com/sr1x3lsr"}}'
import requests
import json
base_url = 'https://raw.githubusercontent.com/Mathpix/api-examples/master/images/'
data = {
'urls': {
'algebra': base_url + 'algebra.jpg',
'inverted': base_url + 'inverted.jpg'
},
'callback': {
'post': 'http://RequestBin.com/1ftatkr1'
}
}
r = requests.post(
"https://api.mathpix.com/v3/batch", data=json.dumps(data),
headers={
'app_id': 'trial',
'app_key': '34f1a4cea0eaca8540c95908b4dc84ab',
'content-type': 'application/json'
},
timeout=30
)
reply = json.loads(r.text)
assert reply.has_key('batch_id')
The callback response is as follows:
{
"reply": {
"batch_id": "11"
},
"result": {
"algebra": {
"detection_list": [],
"detection_map": {
"contains_chart": 0,
"contains_diagram": 0,
"contains_geometry": 0,
"contains_graph": 0,
"contains_table": 0,
"is_inverted": 0,
"is_not_math": 0,
"is_printed": 0
},
"error": "",
"latex": "12 + 5 x - 8 = 12 x - 10",
"latex_confidence": 0.99640350138238,
"latex_list": [],
"position": {
"height": 208,
"top_left_x": 0,
"top_left_y": 0,
"width": 1380
}
},
"inverted": {
"detection_list": [
"is_inverted",
"is_printed"
],
"detection_map": {
"contains_chart": 0,
"contains_diagram": 0,
"contains_geometry": 0,
"contains_graph": 0,
"contains_table": 0,
"is_inverted": 1,
"is_not_math": 0,
"is_printed": 1
},
"error": "",
"latex": "x ^ { 2 } + y ^ { 2 } = 9",
"latex_confidence": 0.99982263230866,
"latex_list": [],
"position": {
"height": 170,
"top_left_x": 48,
"top_left_y": 85,
"width": 544
}
}
}
}
The Mathpix API supports processing multiple images
in a single POST request to a different endpoint: /v3/batch.
The request contains urls
specifying a key-url pair for each image, and callback
that specifies where to send the results. The url in a key-url pair
may either be a string or an object containing the url and
additional image-specific request parameters such as region and formats.
The response contains a unique batch_id value, and
after processing all the images the server posts a message to callback.post
containing result with a key-result pair for each image result and
reply containing the batch_id.
An application may specify a value for callback.reply in the request
to include fields in addition to batch_id in both the response and
result post. However, the server-generated batch_id cannot be overridden.
Applications that wish to provide additional data in the post may specify
a callback.body value. The server will pass this value to the post.
If callback.headers is provided then the server will send the contents
as headers in the post.
To check on the status of a batch request an application may use
GET /v3/batch/:id where :id is the returned batch_id.
Note that the GET request must contain the same app_id and app_key headers
as the batch request. The response body is JSON containing the batch
url keys (array of strings), current results (key-result objects), and
the callback object passed to the original batch request.
The server only keeps batch results for a limited amount of time
after the batch is complete (currently two days but could change in the future).
Long division
{
"detection_map": {
"contains_chart": 0,
"contains_diagram": 0,
"contains_geometry": 0,
"contains_graph": 0,
"contains_table": 0,
"is_inverted": 0,
"is_not_math": 0,
"is_printed": 1
},
"latex": "8 \\longdiv { 7200 }"
}
The following
<script>tags will take care of rendering the\longdivmarkup.
<script type="text/javascript" src="//cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
<script type="text/x-mathjax-config">
MathJax.Hub.Config({
tex2jax: {inlineMath: [['$','$'], ['\\(','\\)']]},
messageStyle: "none",
TeX: {
Macros: {
longdiv: ["{\\overline{\\smash{)}#1}}", 1]
}
}
});
</script>
We use the special markup \longdiv to represent long division; it is the only nonvalid Latex markup we return. Long division is used much like \sqrt which is visually similar.
Latency considerations
The biggest source of latency is image uploads. The speed of a response from Mathpix API servers is roughly proportional to the size of the image. Try to use images under 100kb for maximum speeds. JPEG compression and image downsizing are recommended to ensure lowest possible latencies.
Errors
In addition to the error field that contains a string (en-us locale) describing an error, a Mathpix response has an error_info field with an object providing programmatic information. The fields of this object include id, uniquely specifying the error, message, containing a string similar to the top-level error field, and detail fields specific to the type of error. The table below lists the different errors Mathpix returns.
| Id | Description | Detail fields |
|---|---|---|
| http_unauthorized | Invalid credentials | |
| http_max_requests | Too many requests | count |
| json_syntax | JSON syntax error | |
| image_missing | Missing URL in request body | |
| image_download_error | Error downloading image | url |
| image_decode_error | Cannot decode the image data | |
| image_no_content | No content found in image | |
| image_not_supported | Image is not math or text | |
| image_max_size | Image is too large to process | |
| opts_bad_callback | Bad callback field(s) | post?, reply?, batch_id? |
| opts_unknown_ocr | Unknown ocr option(s) | ocr |
| opts_unknown_format | Unknown format option(s) | formats |
| math_confidence | Low confidence | |
| math_syntax | Unrecognized math | |
| batch_unknown_id | Unknown batch id | batch_id |
| sys_exception | Server error | batch_id? |
The two HTTP errors, http_unauthorized and http_max_requests, return HTTP status codes 401 and 429, respectively. All other errors return status 200.
The json_syntax error occurs if the Mathpix server cannot parse the request body. Error ids that begin with “image_” indicate a problem with the image sent, including a request that does not specify an image, a url that cannot be downloaded, image data that cannot be decoded, an image with no recognizable content, an image with unsupported content such as a diagram, or an image that is too large to process. An image_download_error includes a detail field containing the problematic url. Error ids that begin with “opts_” indicate a problem with options in the request body, such as an unknown ocr or format string, or a callback with a bad post, reply, or batch_id field. An opts_bad_callback error includes a detail field for the bad input. Error ids that begin with “math_” indicate either a below-threshold confidence in the recognized LaTeX or a result that is not correct LaTeX syntax.
A GET request on /v3/batch/:id will return a batch_unknown_id error if there is no record of a batch with the given id initiated by a /v3/batch POST using the same credentials. Note this error can occur if the batch was initiated more than a certain period of time (currently two days) before the GET request.
A sys_exception error means the Mathpix server could not correctly process the request. Please report these errors by sending email to support@mathpix.com.
Vocabulary
Mathpix generates any of the following characters:
| ! | “ | $ | & | ’ |
| ( | ) | * | + | , |
| - | . | / | 0 | 1 |
| 2 | 3 | 4 | 5 | 6 |
| 7 | 8 | 9 | : | ; |
| < | = | > | ? | A |
| B | C | D | E | F |
| G | H | I | J | K |
| L | M | N | O | P |
| Q | R | S | T | U |
| V | W | X | Y | Z |
| [ | \ | ] | ^ | _ |
| a | b | c | d | e |
| f | g | h | i | j |
| k | l | m | n | o |
| p | q | r | s | t |
| u | v | w | x | y |
| z | { | | | } | ~ |
| \% | \\ | \{ | \} | |
| \alpha | \angle | \langle | \rangle | \approx |
| \because | \begin{array} | \beta | \bot | \cap |
| \cdot | \cdots | \chi | \circ | \cong |
| \cup | \dagger | \Delta | \delta | \div |
| \dot | \dots | \ell | \emptyset | \eta |
| \end{array} | \epsilon | \equiv | \exists | \kappa |
| \forall | \frac | \gamma | \Gamma | \geq |
| \hat | \hbar | \hline | \in | \infty |
| \int | \Lambda | \lambda | \lceil | \left( |
| \left. | \left[ | \left\{ | \left| | \Leftrightarrow |
| \leftrightarrow | \lfloor | \leq | \longdiv | \mathcal |
| \mp | \mu | \nabla | \neq | \notin |
| \oint | \Omega | \omega | \operatorna | \oplus |
| \overline | \otimes | \parallel | \partial | \perp |
| \Phi | \phi | \pi | \pm | \prime |
| \prod | \propto | \Psi | \psi | \qquad |
| \quad | \rceil | \rfloor | \rho | \right) |
| \right. | \right\} | \right] | \Rightarrow | \rightarrow |
| \right| | \sigma | \sim | \simeq | \sqrt |
| \square | \star | \sum | \subset | \supset |
| \subseteq | \supseteq | \tau | \text | \therefore |
| \Theta | \theta | \times | \tilde | |
| \varphi | \vdots | \vec | \wedge | \vee |
| \Xi | \xi | \zeta |